2024-09-17T08:35:11.059 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T08:35:11.081 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T08:35:11.199 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238 branch: reef description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7908238' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_STRAY_DAEMON - CEPHADM_FAILED_DAEMON - CEPHADM_AGENT_DOWN log-only-match: - CEPHADM_ sha1: 19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 workunit: branch: reef sha1: 1cbdc9211df088cccbd94423c7cf3a2b85fe432b owner: scheduled_teuthology@teuthology priority: 930 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5234 sha1: 19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 sleep_before_teardown: 0 subset: 46/64 suite: orch suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_1cbdc9211df088cccbd94423c7cf3a2b85fe432b/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1cbdc9211df088cccbd94423c7cf3a2b85fe432b targets: smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGFPB8psmNtv6H4MlcuJROiY3y2WdTA/4xm/Gd2jxbWvUo9+1/eG6v3LeVDJm+PHVFrMJnL/z+dlAezhudCIa/k= smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhzBikEAxEJgaA7eF5jXK3Q0Pldb7s8rYD8B0tILISgFhu8i5gsoSX7dCrTif/MlXnaofdFbAL7XPeVtrkh9B4= tasks: - cephadm: allow_ptrace: false avoid_pacific_features: true cephadm_branch: v16.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v16.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_22:08:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-17T08:35:11.199 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1cbdc9211df088cccbd94423c7cf3a2b85fe432b/qa; will attempt to use it 2024-09-17T08:35:11.200 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1cbdc9211df088cccbd94423c7cf3a2b85fe432b/qa/tasks 2024-09-17T08:35:11.200 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T08:35:11.201 INFO:teuthology.task.internal:Checking packages... 2024-09-17T08:35:11.222 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '19e2a2beb6f41a3ab62650e38614eb29b59dcbb9' 2024-09-17T08:35:11.222 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T08:35:11.222 INFO:teuthology.packaging:ref: None 2024-09-17T08:35:11.222 INFO:teuthology.packaging:tag: None 2024-09-17T08:35:11.222 INFO:teuthology.packaging:branch: reef 2024-09-17T08:35:11.222 INFO:teuthology.packaging:sha1: 19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T08:35:11.222 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-17T08:35:11.484 INFO:teuthology.task.internal:Found packages for ceph version 18.2.4-855.g19e2a2be 2024-09-17T08:35:11.486 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T08:35:11.493 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T08:35:11.493 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T08:35:11.501 INFO:teuthology.task.internal:Saving configuration 2024-09-17T08:35:11.519 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T08:35:11.526 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T08:35:11.552 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 08:29:09.997549', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGFPB8psmNtv6H4MlcuJROiY3y2WdTA/4xm/Gd2jxbWvUo9+1/eG6v3LeVDJm+PHVFrMJnL/z+dlAezhudCIa/k='} 2024-09-17T08:35:11.575 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 08:29:09.995813', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGhzBikEAxEJgaA7eF5jXK3Q0Pldb7s8rYD8B0tILISgFhu8i5gsoSX7dCrTif/MlXnaofdFbAL7XPeVtrkh9B4='} 2024-09-17T08:35:11.575 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T08:35:11.582 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-17T08:35:11.582 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-17T08:35:11.582 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T08:35:11.641 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f6d0a9b3ac0>, signals=[15]) 2024-09-17T08:35:11.641 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T08:35:11.647 INFO:teuthology.task.internal:Opening connections... 2024-09-17T08:35:11.647 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2024-09-17T08:35:11.649 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:35:11.726 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-09-17T08:35:11.727 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:35:11.799 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T08:35:11.806 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2024-09-17T08:35:11.825 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2024-09-17T08:35:11.825 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2024-09-17T08:35:11.883 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="CentOS Stream" 2024-09-17T08:35:11.883 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="9" 2024-09-17T08:35:11.883 INFO:teuthology.orchestra.run.smithi027.stdout:ID="centos" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="rhel fedora" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="9" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el9" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:LOGO="fedora-logo-icon" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://centos.org/" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T08:35:11.884 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T08:35:11.885 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2024-09-17T08:35:12.111 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-09-17T08:35:12.130 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-09-17T08:35:12.130 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="CentOS Stream" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="9" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:ID="centos" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="rhel fedora" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="9" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el9" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-09-17T08:35:12.188 INFO:teuthology.orchestra.run.smithi105.stdout:LOGO="fedora-logo-icon" 2024-09-17T08:35:12.189 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T08:35:12.189 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://centos.org/" 2024-09-17T08:35:12.189 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T08:35:12.189 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T08:35:12.189 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T08:35:12.189 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-09-17T08:35:12.214 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T08:35:12.224 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T08:35:12.230 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T08:35:12.230 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2024-09-17T08:35:12.232 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-09-17T08:35:12.248 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T08:35:12.255 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T08:35:12.255 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2024-09-17T08:35:12.290 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-09-17T08:35:12.386 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T08:35:12.434 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T08:35:12.455 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T08:35:12.455 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T08:35:12.455 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T08:35:12.455 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T08:35:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T08:35:12.472 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-17T08:35:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T08:35:12.473 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-09-17T08:35:12.530 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-503.el9.x86_64 2024-09-17T08:35:12.530 INFO:teuthology.task.kernel:Running kernel on smithi105: 5.14.0-503.el9.x86_64 2024-09-17T08:35:12.530 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum install -y kernel 2024-09-17T08:35:12.532 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-503.el9.x86_64 2024-09-17T08:35:12.532 INFO:teuthology.task.kernel:Running kernel on smithi027: 5.14.0-503.el9.x86_64 2024-09-17T08:35:12.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2024-09-17T08:35:14.766 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.3 MB 00:00 2024-09-17T08:35:16.525 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-17T08:35:16.662 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - BaseOS 3.5 MB/s | 8.3 MB 00:02 2024-09-17T08:35:18.504 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - AppStream 28 MB/s | 20 MB 00:00 2024-09-17T08:35:20.776 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-17T08:35:22.369 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 50 kB/s | 19 kB 00:00 2024-09-17T08:35:23.080 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - CRB 9.5 MB/s | 6.5 MB 00:00 2024-09-17T08:35:23.523 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-17T08:35:25.339 INFO:teuthology.orchestra.run.smithi027.stdout:CentOS Stream 9 - Extras packages 19 kB/s | 19 kB 00:00 2024-09-17T08:35:26.185 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 32 MB/s | 23 MB 00:00 2024-09-17T08:35:28.954 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-17T08:35:30.852 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T08:35:30.852 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T08:35:30.954 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-17T08:35:30.962 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T08:35:30.962 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-17T08:35:30.962 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 79 M 2024-09-17T08:35:30.963 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 126 M 2024-09-17T08:35:30.964 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-17T08:35:31.205 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 330 kB/s | 36 kB 00:00 2024-09-17T08:35:31.833 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-09-17T08:35:33.505 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 6.5 MB/s | 16 MB 00:02 2024-09-17T08:35:33.806 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T08:35:33.806 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T08:35:33.886 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T08:35:33.901 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout:Install 4 Packages 2024-09-17T08:35:33.902 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:35:33.903 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 79 M 2024-09-17T08:35:33.903 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 126 M 2024-09-17T08:35:33.903 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2024-09-17T08:35:34.076 INFO:teuthology.orchestra.run.smithi027.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 282 kB/s | 36 kB 00:00 2024-09-17T08:35:34.718 INFO:teuthology.orchestra.run.smithi027.stdout:(2/4): kernel-modules-core-5.14.0-508.el9.x86_6 45 MB/s | 29 MB 00:00 2024-09-17T08:35:35.005 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 7.5 MB/s | 29 MB 00:03 2024-09-17T08:35:35.035 INFO:teuthology.orchestra.run.smithi027.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 32 MB/s | 35 MB 00:01 2024-09-17T08:35:35.860 INFO:teuthology.orchestra.run.smithi027.stdout:(4/4): kernel-core-5.14.0-508.el9.x86_64.rpm 8.2 MB/s | 16 MB 00:01 2024-09-17T08:35:35.860 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2024-09-17T08:35:35.861 INFO:teuthology.orchestra.run.smithi027.stdout:Total 40 MB/s | 79 MB 00:01 2024-09-17T08:35:36.014 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 7.0 MB/s | 35 MB 00:04 2024-09-17T08:35:36.014 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-17T08:35:36.014 INFO:teuthology.orchestra.run.smithi105.stdout:Total 16 MB/s | 79 MB 00:05 2024-09-17T08:35:36.573 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2024-09-17T08:35:36.702 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-17T08:35:36.972 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2024-09-17T08:35:36.972 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2024-09-17T08:35:37.051 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-17T08:35:37.051 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-17T08:35:37.990 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2024-09-17T08:35:37.991 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2024-09-17T08:35:38.037 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-17T08:35:38.037 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-17T08:35:40.460 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-17T08:35:40.562 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2024-09-17T08:35:41.099 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T08:35:41.197 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:35:41.240 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T08:35:41.321 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:35:43.966 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:35:44.101 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:35:44.209 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:35:44.339 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:35:48.905 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:35:48.952 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:35:49.567 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:35:49.607 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:35:54.093 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:35:54.840 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:32.622 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:32.661 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:33.026 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:33.026 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T08:36:33.027 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:36:33.027 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T08:36:33.878 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T08:36:33.879 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T08:36:33.879 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:36:33.879 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-17T08:36:34.227 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T08:36:34.315 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:34.316 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T08:36:34.316 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T08:36:34.316 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T08:36:34.316 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T08:36:34.316 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-17T08:36:34.316 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi105.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T08:36:34.317 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T08:36:34.375 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:34.375 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T08:36:34.375 DEBUG:teuthology.orchestra.run.smithi105:> sudo rpm -qi grub2-tools 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Name : grub2-tools 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Epoch : 1 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Version : 2.06 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Release : 82.el9 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Architecture: x86_64 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Group : Unspecified 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Size : 8274670 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:License : GPLv3+ 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T08:36:34.407 INFO:teuthology.orchestra.run.smithi105.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Packager : builder@centos.org 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Vendor : CentOS 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Summary : Support tools for GRUB. 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:Description : 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout:hardware devices. 2024-09-17T08:36:34.408 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:36:34.409 INFO:teuthology.orchestra.run.smithi105.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T08:36:34.409 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T08:36:34.409 INFO:teuthology.task.kernel:Updating grub on smithi105 to boot 5.14.0-508.el9.x86_64 2024-09-17T08:36:34.410 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T08:36:34.719 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:34.762 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:35.043 INFO:teuthology.orchestra.run.smithi105.stderr:Generating grub configuration file ... 2024-09-17T08:36:35.154 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:35.154 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T08:36:35.154 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T08:36:35.155 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T08:36:35.965 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T08:36:35.966 INFO:teuthology.orchestra.run.smithi027.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T08:36:35.966 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:36:35.966 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2024-09-17T08:36:36.320 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T08:36:36.367 INFO:teuthology.orchestra.run.smithi105.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T08:36:36.397 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:36.398 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T08:36:36.398 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T08:36:36.398 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T08:36:36.399 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T08:36:36.399 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-17T08:36:36.399 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi027.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T08:36:36.399 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T08:36:36.401 INFO:teuthology.orchestra.run.smithi105.stderr:done 2024-09-17T08:36:36.403 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-17T08:36:36.419 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.yVlVWswvQS 2024-09-17T08:36:36.419 DEBUG:teuthology.orchestra.run.smithi105:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.yVlVWswvQS 2024-09-17T08:36:36.459 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T08:36:36.459 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T08:36:36.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo rpm -qi grub2-tools 2024-09-17T08:36:36.494 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.yVlVWswvQS 2024-09-17T08:36:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:Name : grub2-tools 2024-09-17T08:36:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:Epoch : 1 2024-09-17T08:36:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:Version : 2.06 2024-09-17T08:36:36.497 INFO:teuthology.orchestra.run.smithi027.stdout:Release : 82.el9 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Architecture: x86_64 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Group : Unspecified 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Size : 8274670 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:License : GPLv3+ 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T08:36:36.498 INFO:teuthology.orchestra.run.smithi027.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:Packager : builder@centos.org 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:Vendor : CentOS 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:Summary : Support tools for GRUB. 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:Description : 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:hardware devices. 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:36:36.499 INFO:teuthology.orchestra.run.smithi027.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T08:36:36.500 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T08:36:36.500 INFO:teuthology.task.kernel:Updating grub on smithi027 to boot 5.14.0-508.el9.x86_64 2024-09-17T08:36:36.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T08:36:36.643 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.yVlVWswvQS is 6KB 2024-09-17T08:36:36.692 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.yVlVWswvQS 2024-09-17T08:36:36.707 DEBUG:teuthology.orchestra.run.smithi105:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T08:36:36.775 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T08:36:36.775 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T08:36:36.775 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T08:36:36.775 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T08:36:36.777 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T08:36:36.936 DEBUG:teuthology.orchestra.run.smithi105:> sudo shutdown -r now 2024-09-17T08:36:37.203 INFO:teuthology.orchestra.run.smithi027.stderr:Generating grub configuration file ... 2024-09-17T08:36:38.626 INFO:teuthology.orchestra.run.smithi027.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T08:36:38.643 INFO:teuthology.orchestra.run.smithi027.stderr:done 2024-09-17T08:36:38.645 DEBUG:teuthology.orchestra.run.smithi027:> mktemp 2024-09-17T08:36:38.701 INFO:teuthology.orchestra.run.smithi027.stdout:/tmp/tmp.J1jueDN7b0 2024-09-17T08:36:38.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.J1jueDN7b0 2024-09-17T08:36:38.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0666 /tmp/tmp.J1jueDN7b0 2024-09-17T08:36:38.910 DEBUG:teuthology.orchestra.remote:smithi027:/tmp/tmp.J1jueDN7b0 is 6KB 2024-09-17T08:36:38.959 DEBUG:teuthology.orchestra.run.smithi027:> rm -fr /tmp/tmp.J1jueDN7b0 2024-09-17T08:36:38.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T08:36:39.039 INFO:teuthology.orchestra.run.smithi027.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T08:36:39.039 INFO:teuthology.orchestra.run.smithi027.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T08:36:39.039 INFO:teuthology.orchestra.run.smithi027.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T08:36:39.039 INFO:teuthology.orchestra.run.smithi027.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T08:36:39.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T08:36:39.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo shutdown -r now 2024-09-17T08:37:06.967 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-17T08:37:06.967 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:37:06.969 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:09.188 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-17T08:37:09.435 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:37:09.436 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:25.437 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-17T08:37:28.001 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-09-17T08:37:34.445 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:37:34.446 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:37.004 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:37:37.005 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:37.505 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.105 2024-09-17T08:37:40.061 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-09-17T08:37:49.515 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:37:49.516 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:52.065 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:37:52.065 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:37:55.133 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-09-17T08:38:10.145 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:38:10.146 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:38:13.217 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.27 2024-09-17T08:38:21.504 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-17T08:38:22.058 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:38:22.059 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T08:38:22.059 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-17T08:38:22.102 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-508.el9.x86_64 2024-09-17T08:38:22.102 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T08:38:22.103 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T08:38:22.103 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T08:38:23.103 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-17T08:38:23.104 DEBUG:teuthology.orchestra.run.smithi105:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T08:38:23.267 INFO:teuthology.orchestra.run.smithi105.stdout:ttyS1 2024-09-17T08:38:23.484 DEBUG:teuthology.parallel:result is None 2024-09-17T08:38:31.218 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:38:31.219 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:39:03.065 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-09-17T08:39:03.384 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:39:03.384 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T08:39:03.385 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2024-09-17T08:39:03.401 INFO:teuthology.orchestra.run.smithi027.stdout:5.14.0-508.el9.x86_64 2024-09-17T08:39:03.401 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T08:39:03.401 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T08:39:03.402 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T08:39:04.402 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-17T08:39:04.403 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T08:39:04.479 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2024-09-17T08:39:04.504 DEBUG:teuthology.parallel:result is None 2024-09-17T08:39:04.504 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T08:39:04.512 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T08:39:04.512 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T08:39:04.514 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T08:39:04.533 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T08:39:04.539 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T08:39:04.547 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T08:39:04.547 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T08:39:04.573 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T08:39:04.614 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T08:39:04.620 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T08:39:04.621 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T08:39:04.652 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T08:39:04.683 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T08:39:04.695 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T08:39:04.697 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T08:39:04.708 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T08:39:04.710 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T08:39:04.716 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T08:39:04.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T08:39:04.739 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T08:39:04.765 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T08:39:04.774 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T08:39:04.775 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T08:39:04.808 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T08:39:04.824 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T08:39:04.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T08:39:04.956 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T08:39:05.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T08:39:05.082 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:39:05.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T08:39:05.148 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T08:39:05.192 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T08:39:05.241 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T08:39:05.324 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T08:39:05.365 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:39:05.365 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T08:39:05.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2024-09-17T08:39:05.432 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-09-17T08:39:05.485 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T08:39:05.514 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T08:39:05.986 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T08:39:05.996 INFO:teuthology.task.internal:Starting timer... 2024-09-17T08:39:05.996 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T08:39:06.024 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T08:39:06.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2024-09-17T08:39:06.113 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2024-09-17T08:39:06.115 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-09-17T08:39:06.161 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-09-17T08:39:06.163 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T08:39:06.163 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2024-09-17T08:39:06.206 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2024-09-17T08:39:06.206 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-09-17T08:39:06.227 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-09-17T08:39:06.227 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi027.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi105.front.sepia.ceph.com': 'permissive'} 2024-09-17T08:39:06.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T08:39:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T08:39:06.276 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T08:39:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T08:39:06.302 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T08:39:06.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2024-09-17T08:39:06.346 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-09-17T08:39:06.376 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T08:39:06.385 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-17T08:39:06.388 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-17T08:39:06.500 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T08:39:06.511 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T08:39:06.512 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi027.front.sepia.ceph.com,smithi105.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T08:42:55.980 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi027.front.sepia.ceph.com'), Remote(name='ubuntu@smithi105.front.sepia.ceph.com')] 2024-09-17T08:42:55.982 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:42:55.984 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:42:56.068 DEBUG:teuthology.orchestra.run.smithi027:> true 2024-09-17T08:42:56.149 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2024-09-17T08:42:56.150 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:42:56.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T08:42:56.228 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-17T08:42:56.305 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-17T08:42:56.305 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T08:42:56.318 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T08:42:56.318 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T08:42:56.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T08:42:56.321 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T08:42:56.322 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T08:42:56.352 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T08:42:56.368 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T08:42:56.375 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T08:42:56.390 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T08:42:56.395 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: ntpd: command not found 2024-09-17T08:42:56.407 INFO:teuthology.orchestra.run.smithi027.stdout:506 Cannot talk to daemon 2024-09-17T08:42:56.415 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-09-17T08:42:56.423 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T08:42:56.426 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-09-17T08:42:56.437 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T08:42:56.440 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T08:42:56.453 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T08:42:56.496 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-09-17T08:42:56.499 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T08:42:56.499 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-09-17T08:42:56.499 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.499 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.500 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.500 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.508 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-17T08:42:56.513 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T08:42:56.513 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-17T08:42:56.513 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.513 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.514 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.514 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T08:42:56.514 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T08:42:56.606 INFO:tasks.cephadm:Config: {'allow_ptrace': False, 'avoid_pacific_features': True, 'cephadm_branch': 'v16.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v16.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': True}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_STRAY_DAEMON', 'CEPHADM_FAILED_DAEMON', 'CEPHADM_AGENT_DOWN'], 'log-only-match': ['CEPHADM_'], 'sha1': '19e2a2beb6f41a3ab62650e38614eb29b59dcbb9'} 2024-09-17T08:42:56.606 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v16.2.0 2024-09-17T08:42:56.607 INFO:tasks.cephadm:Cluster fsid is d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:42:56.607 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T08:42:56.607 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.27', 'mon.c': '[v2:172.21.15.27:3301,v1:172.21.15.27:6790]', 'mon.b': '172.21.15.105'} 2024-09-17T08:42:56.608 INFO:tasks.cephadm:First mon is mon.a on smithi027 2024-09-17T08:42:56.608 INFO:tasks.cephadm:First mgr is y 2024-09-17T08:42:56.608 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T08:42:56.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo hostname $(hostname -s) 2024-09-17T08:42:56.635 DEBUG:teuthology.orchestra.run.smithi105:> sudo hostname $(hostname -s) 2024-09-17T08:42:56.661 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v16.2.0)... 2024-09-17T08:42:56.661 DEBUG:teuthology.orchestra.run.smithi027:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:56.899 INFO:teuthology.orchestra.run.smithi027.stdout:-rw-r--r--. 1 ubuntu ubuntu 288386 Sep 17 08:42 /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:56.900 DEBUG:teuthology.orchestra.run.smithi105:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:57.024 INFO:teuthology.orchestra.run.smithi105.stdout:-rw-r--r--. 1 ubuntu ubuntu 288386 Sep 17 08:42 /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:57.025 DEBUG:teuthology.orchestra.run.smithi027:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:57.042 DEBUG:teuthology.orchestra.run.smithi105:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T08:42:57.068 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v16.2.0 on all hosts... 2024-09-17T08:42:57.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 pull 2024-09-17T08:42:57.085 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 pull 2024-09-17T08:42:57.942 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-09-17T08:42:57.976 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-09-17T08:43:09.872 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_version": "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)", 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: "image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: "repo_digests": [ 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.io/ceph/ceph@sha256:9b04c0f15704c49591640a37c7adfd40ffad0a4b42fecb950c3407687cb4f29a" 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-17T08:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-17T08:43:10.098 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:43:10.098 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph_version": "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)", 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout: "image_id": "24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15", 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout: "repo_digests": [ 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000", 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.io/ceph/ceph@sha256:9b04c0f15704c49591640a37c7adfd40ffad0a4b42fecb950c3407687cb4f29a" 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-09-17T08:43:10.099 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:43:10.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo mkdir -p /etc/ceph 2024-09-17T08:43:10.161 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph 2024-09-17T08:43:10.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 777 /etc/ceph 2024-09-17T08:43:10.214 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /etc/ceph 2024-09-17T08:43:10.253 INFO:tasks.cephadm:Writing seed config... 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = True 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T08:43:10.255 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T08:43:10.256 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:43:10.256 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T08:43:10.271 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = d61dd7c6-74d0-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = True [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T08:43:10.271 DEBUG:teuthology.orchestra.run.smithi027:mon.a> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service 2024-09-17T08:43:10.314 DEBUG:teuthology.orchestra.run.smithi027:mgr.y> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service 2024-09-17T08:43:10.356 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T08:43:10.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 -v bootstrap --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.27 && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T08:43:10.549 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman --version 2024-09-17T08:43:10.570 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: podman version 5.2.2 2024-09-17T08:43:10.571 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying podman|docker is present... 2024-09-17T08:43:10.572 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman --version 2024-09-17T08:43:10.590 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: podman version 5.2.2 2024-09-17T08:43:10.592 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying lvm2 is present... 2024-09-17T08:43:10.593 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying time synchronization is in place... 2024-09-17T08:43:10.593 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled chrony.service 2024-09-17T08:43:10.600 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T08:43:10.600 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active chrony.service 2024-09-17T08:43:10.606 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:43:10.607 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled chronyd.service 2024-09-17T08:43:10.614 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: enabled 2024-09-17T08:43:10.614 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active chronyd.service 2024-09-17T08:43:10.620 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: active 2024-09-17T08:43:10.621 INFO:teuthology.orchestra.run.smithi027.stderr:Unit chronyd.service is enabled and running 2024-09-17T08:43:10.621 INFO:teuthology.orchestra.run.smithi027.stderr:Repeating the final host check... 2024-09-17T08:43:10.621 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman --version 2024-09-17T08:43:10.640 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: podman version 5.2.2 2024-09-17T08:43:10.642 INFO:teuthology.orchestra.run.smithi027.stderr:podman|docker (/bin/podman) is present 2024-09-17T08:43:10.642 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl is present 2024-09-17T08:43:10.642 INFO:teuthology.orchestra.run.smithi027.stderr:lvcreate is present 2024-09-17T08:43:10.642 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled chrony.service 2024-09-17T08:43:10.648 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T08:43:10.649 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active chrony.service 2024-09-17T08:43:10.655 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:43:10.655 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled chronyd.service 2024-09-17T08:43:10.662 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: enabled 2024-09-17T08:43:10.663 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active chronyd.service 2024-09-17T08:43:10.669 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: active 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Unit chronyd.service is enabled and running 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Host looks OK 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Cluster fsid: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Acquiring lock 140053671299104 on /run/cephadm/d61dd7c6-74d0-11ef-bceb-c7b262605968.lock 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Lock 140053671299104 acquired on /run/cephadm/d61dd7c6-74d0-11ef-bceb-c7b262605968.lock 2024-09-17T08:43:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying IP 172.21.15.27 port 3300 ... 2024-09-17T08:43:10.671 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying IP 172.21.15.27 port 6789 ... 2024-09-17T08:43:10.671 INFO:teuthology.orchestra.run.smithi027.stderr:Base mon IP is 172.21.15.27, final addrv is [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-09-17T08:43:10.671 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /sbin/ip route ls 2024-09-17T08:43:10.673 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.27 metric 100 2024-09-17T08:43:10.673 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.27 metric 100 2024-09-17T08:43:10.674 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /sbin/ip -6 route ls 2024-09-17T08:43:10.676 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T08:43:10.676 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-17T08:43:10.676 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1772sec hoplimit 64 pref medium 2024-09-17T08:43:10.676 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /sbin/ip -6 addr ls 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:395a/64 scope link 2024-09-17T08:43:10.679 INFO:teuthology.orchestra.run.smithi027.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T08:43:10.680 INFO:teuthology.orchestra.run.smithi027.stderr:Mon IP 172.21.15.27 is in CIDR network 172.21.0.0/20 2024-09-17T08:43:10.680 INFO:teuthology.orchestra.run.smithi027.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T08:43:10.681 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.io/ceph/ceph:v16.2.0... 2024-09-17T08:43:10.681 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman pull quay.io/ceph/ceph:v16.2.0 2024-09-17T08:43:10.784 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v16.2.0... 2024-09-17T08:43:11.217 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Getting image source signatures 2024-09-17T08:43:11.217 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:958e3c20ad47ec7f3ec0446a4a630f2285dd6b0a4c969f4f702f7c2e78f3d68b 2024-09-17T08:43:11.217 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying blob sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621 2024-09-17T08:43:11.217 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Copying config sha256:24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 2024-09-17T08:43:11.385 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T08:43:11.521 INFO:teuthology.orchestra.run.smithi027.stderr:/bin/podman: 24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 2024-09-17T08:43:11.524 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --version 2024-09-17T08:43:11.875 INFO:teuthology.orchestra.run.smithi027.stderr:ceph: ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable) 2024-09-17T08:43:12.437 INFO:teuthology.orchestra.run.smithi027.stderr:Ceph version: ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable) 2024-09-17T08:43:12.437 INFO:teuthology.orchestra.run.smithi027.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T08:43:12.437 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 -c %u %g /var/lib/ceph 2024-09-17T08:43:12.811 INFO:teuthology.orchestra.run.smithi027.stderr:stat: 167 167 2024-09-17T08:43:13.344 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial keys... 2024-09-17T08:43:13.344 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-09-17T08:43:13.769 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAhQelmEalmJxAAJsV3Q7EEcqQe1CmenNpsMw== 2024-09-17T08:43:14.291 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-09-17T08:43:14.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAiQelm/QXDJBAA1sWVxRDaezFFfuName41hg== 2024-09-17T08:43:15.163 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 quay.io/ceph/ceph:v16.2.0 --gen-print-key 2024-09-17T08:43:15.588 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-authtool: AQAjQelmOFMbHRAAV1aUWbP7kQx+PZX/jeLtiQ== 2024-09-17T08:43:16.180 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial monmap... 2024-09-17T08:43:16.180 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/monmaptool --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /tmp/ceph-tmp4mspaa15:/tmp/monmap:z quay.io/ceph/ceph:v16.2.0 --create --clobber --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --addv a [v2:172.21.15.27:3300,v1:172.21.15.27:6789] /tmp/monmap 2024-09-17T08:43:16.594 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T08:43:16.594 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:16.594 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T08:43:17.034 INFO:teuthology.orchestra.run.smithi027.stderr:monmaptool for a [v2:172.21.15.27:3300,v1:172.21.15.27:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T08:43:17.034 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: set fsid to d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:17.034 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T08:43:17.034 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:43:17.035 INFO:teuthology.orchestra.run.smithi027.stderr:Creating mon... 2024-09-17T08:43:17.036 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-mon --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a:/var/lib/ceph/mon/ceph-a:z -v /tmp/ceph-tmprjiksg2u:/tmp/keyring:z -v /tmp/ceph-tmp4mspaa15:/tmp/monmap:z quay.io/ceph/ceph:v16.2.0 --mkfs -i a --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2024-09-17T08:43:17.460 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.400+0000 7f93c82d1700 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-17T08:43:17.460 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.400+0000 7f93c82d1700 1 imported monmap: 2024-09-17T08:43:17.460 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: epoch 0 2024-09-17T08:43:17.460 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: last_changed 2024-09-17T08:43:16.507114+0000 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: created 2024-09-17T08:43:16.507114+0000 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: election_strategy: 1 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon.a 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.400+0000 7f93c82d1700 0 /usr/bin/ceph-mon: set fsid to d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: RocksDB version: 6.8.1 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T08:43:17.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Compile date Mar 30 2021 2024-09-17T08:43:17.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: DB SUMMARY 2024-09-17T08:43:17.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.error_if_exists: 0 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.create_if_missing: 1 2024-09-17T08:43:17.496 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.paranoid_checks: 1 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.env: 0x55ba2e59d1c0 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.fs: Posix File System 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.info_log: 0x55ba302f6b80 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.statistics: (nil) 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.use_fsync: 0 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_log_file_size: 0 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.allow_fallocate: 1 2024-09-17T08:43:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.use_direct_reads: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.create_missing_column_families: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.db_log_dir: 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_subcompactions: 1 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_background_flushes: -1 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T08:43:17.498 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.advise_random_on_open: 1 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.write_buffer_manager: 0x55ba30301410 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.rate_limiter: (nil) 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-17T08:43:17.499 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.unordered_write: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.row_cache: None 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.wal_filter: None 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.preserve_deletes: 0 2024-09-17T08:43:17.500 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.two_write_queues: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.manual_wal_flush: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.atomic_flush: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.log_readahead_size: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_background_jobs: 2 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_background_compactions: -1 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T08:43:17.501 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_total_wal_size: 0 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.max_open_files: -1 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.bytes_per_sync: 0 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Compression algorithms supported: 2024-09-17T08:43:17.502 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kZSTD supported: 0 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kXpressCompression supported: 0 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kLZ4Compression supported: 1 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kBZip2Compression supported: 0 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kZlibCompression supported: 1 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: kSnappyCompression supported: 1 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.401+0000 7f93c82d1700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.402+0000 7f93c82d1700 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.503 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.merge_operator: 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_filter: None 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_filter_factory: None 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ba302100d0) 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T08:43:17.504 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: index_type: 0 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: index_shortening: 1 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: checksum: 1 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_cache: 0x55ba30246d10 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_cache_options: 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: capacity : 536870912 2024-09-17T08:43:17.505 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_size: 4096 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: partition_filters: 0 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T08:43:17.506 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: verify_compression: 0 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: format_version: 2 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: block_align: 0 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression: NoCompression 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.num_levels: 7 2024-09-17T08:43:17.507 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.level: 32767 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T08:43:17.508 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compression_opts.enabled: false 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T08:43:17.509 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.arena_block_size: 4194304 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T08:43:17.510 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.table_properties_collectors: 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.inplace_update_support: 0 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.bloom_locality: 0 2024-09-17T08:43:17.511 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.max_successive_merges: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.force_consistency_checks: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.ttl: 2592000 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.403+0000 7f93c82d1700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.404+0000 7f93c82d1700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.404+0000 7f93c82d1700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.404+0000 7f93c82d1700 4 rocksdb: DB pointer 0x55ba3030b800 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.405+0000 7f93b116c700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-09-17T08:43:17.512 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.405+0000 7f93b116c700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:17.513 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:43:17.514 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:43:17.516 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.406+0000 7f93c82d1700 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.406+0000 7f93c82d1700 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2024-09-17T08:43:17.517 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph-mon: debug 2024-09-17T08:43:17.406+0000 7f93c82d1700 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-09-17T08:43:18.057 INFO:teuthology.orchestra.run.smithi027.stderr:create mon.a on 2024-09-17T08:43:18.057 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:18.063 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl disable ceph.target 2024-09-17T08:43:18.271 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl enable ceph.target 2024-09-17T08:43:18.278 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T08:43:18.414 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl start ceph.target 2024-09-17T08:43:18.421 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl enable ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target 2024-09-17T08:43:18.429 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target. 2024-09-17T08:43:18.430 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target. 2024-09-17T08:43:18.566 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl start ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target 2024-09-17T08:43:18.586 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl daemon-reload 2024-09-17T08:43:18.728 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T08:43:18.744 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl reset-failed ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T08:43:18.749 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to reset failed state of unit ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-17T08:43:18.750 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl enable ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T08:43:18.758 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target.wants/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@.service. 2024-09-17T08:43:18.890 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl start ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T08:43:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.983718+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T08:43:20.800 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled firewalld.service 2024-09-17T08:43:20.810 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-09-17T08:43:20.811 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active firewalld.service 2024-09-17T08:43:20.819 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:43:20.820 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-09-17T08:43:20.820 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T08:43:20.821 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[]>. firewalld.service is not available 2024-09-17T08:43:20.821 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mon to start... 2024-09-17T08:43:20.821 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mon... 2024-09-17T08:43:20.821 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a:/var/lib/ceph/mon/ceph-a:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status 2024-09-17T08:43:21.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 bash[23299]: 7d06524e5c5a9d79a6dc6c97d6c5312ffb7584acaca86c8d461274201cfef84b 2024-09-17T08:43:21.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 systemd[1]: Started Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:43:21.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.988047+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T08:43:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.988552+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} 2024-09-17T08:43:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.991628+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-09-17T08:43:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.995567+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-17T08:43:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: cluster 2024-09-17T08:43:19.995982+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-09-17T08:43:24.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: audit 2024-09-17T08:43:23.834962+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.27:0/996696528' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: cluster: 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: id: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: services: 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 3s) 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: data: 2024-09-17T08:43:24.578 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T08:43:24.579 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T08:43:24.579 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T08:43:24.579 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: pgs: 2024-09-17T08:43:24.579 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:29.416 INFO:teuthology.orchestra.run.smithi027.stderr:mon is available 2024-09-17T08:43:29.416 INFO:teuthology.orchestra.run.smithi027.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T08:43:29.417 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z -v /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a:/var/lib/ceph/mon/ceph-a:z quay.io/ceph/ceph:v16.2.0 config assimilate-conf -i /var/lib/ceph/mon/ceph-a/config 2024-09-17T08:43:29.963 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:29.963 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [global] 2024-09-17T08:43:29.963 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: fsid = d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:29.963 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: log_to_journald = false 2024-09-17T08:43:29.963 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.27:3300,v1:172.21.15.27:6789] 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [mgr] 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = True 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [mon] 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_warn_on_insecure_global_id_reclaim = false 2024-09-17T08:43:29.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-17T08:43:29.965 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:29.965 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: [osd] 2024-09-17T08:43:29.965 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T08:43:29.965 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_memory_target_autotune = true 2024-09-17T08:43:29.965 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T08:43:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: audit 2024-09-17T08:43:29.958362+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.27:0/3250735255' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T08:43:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: audit 2024-09-17T08:43:29.960847+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.27:0/3250735255' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T08:43:30.463 INFO:teuthology.orchestra.run.smithi027.stderr:Generating new minimal ceph.conf... 2024-09-17T08:43:30.463 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z -v /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a:/var/lib/ceph/mon/ceph-a:z quay.io/ceph/ceph:v16.2.0 config generate-minimal-conf -o /var/lib/ceph/mon/ceph-a/config 2024-09-17T08:43:31.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: audit 2024-09-17T08:43:30.927029+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.27:0/2618393449' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:43:31.436 INFO:teuthology.orchestra.run.smithi027.stderr:Restarting the monitor... 2024-09-17T08:43:31.436 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl restart ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T08:43:31.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 systemd[1]: Stopping Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:43:31.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: debug 2024-09-17T08:43:31.586+0000 7fd0e159a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-17T08:43:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23313]: debug 2024-09-17T08:43:31.586+0000 7fd0e159a700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T08:43:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 podman[23539]: 2024-09-17 08:43:31.617375278 +0000 UTC m=+0.147933235 container died 7d06524e5c5a9d79a6dc6c97d6c5312ffb7584acaca86c8d461274201cfef84b (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Base Image, RELEASE=HEAD, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau ) 2024-09-17T08:43:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 podman[23539]: 2024-09-17 08:43:31.927514814 +0000 UTC m=+0.458072773 container cleanup 7d06524e5c5a9d79a6dc6c97d6c5312ffb7584acaca86c8d461274201cfef84b (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau , CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T08:43:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:31 smithi027 podman[23539]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a 2024-09-17T08:43:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 podman[23551]: 2024-09-17 08:43:32.078473801 +0000 UTC m=+0.458024439 container remove 7d06524e5c5a9d79a6dc6c97d6c5312ffb7584acaca86c8d461274201cfef84b (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, RELEASE=HEAD, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.name=CentOS Base Image, maintainer=Dimitri Savineau , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204) 2024-09-17T08:43:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-17T08:43:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 systemd[1]: Stopped Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:43:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 systemd[1]: Starting Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:43:32.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 podman[23594]: 2024-09-17 08:43:32.172363977 +0000 UTC m=+0.017713535 image pull 24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 quay.io/ceph/ceph:v16.2.0 2024-09-17T08:43:32.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 podman[23594]: 2024-09-17 08:43:32.288044802 +0000 UTC m=+0.133394393 container create ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, RELEASE=HEAD, maintainer=Dimitri Savineau , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.vendor=CentOS, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, GIT_BRANCH=HEAD) 2024-09-17T08:43:32.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 podman[23594]: 2024-09-17 08:43:32.497638936 +0000 UTC m=+0.342988479 container init ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, maintainer=Dimitri Savineau , ceph=True, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T08:43:32.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 podman[23594]: 2024-09-17 08:43:32.501003851 +0000 UTC m=+0.346353394 container start ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.vendor=CentOS, maintainer=Dimitri Savineau , ceph=True, org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, org.label-schema.license=GPLv2) 2024-09-17T08:43:32.634 INFO:teuthology.orchestra.run.smithi027.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T08:43:32.634 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mon public_network 172.21.0.0/20 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.521+0000 7fb430265700 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.521+0000 7fb430265700 0 ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable), process ceph-mon, pid 2 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.521+0000 7fb430265700 0 pidfile_write: ignore empty --pid-file 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.525+0000 7fb430265700 0 load: jerasure load: lrc load: isa 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: RocksDB version: 6.8.1 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Compile date Mar 30 2021 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: DB SUMMARY 2024-09-17T08:43:32.871 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: CURRENT file: CURRENT 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: IDENTITY file: IDENTITY 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 87076 ; 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.872 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.error_if_exists: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.create_if_missing: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.paranoid_checks: 1 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.env: 0x558a8639d1c0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.fs: Posix File System 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.info_log: 0x558a8730a640 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.statistics: (nil) 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.use_fsync: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_log_file_size: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-17T08:43:32.873 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.allow_fallocate: 1 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.use_direct_reads: 0 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.create_missing_column_families: 0 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.db_log_dir: 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_subcompactions: 1 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_background_flushes: -1 2024-09-17T08:43:32.874 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.advise_random_on_open: 1 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.write_buffer_manager: 0x558a873a1e90 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T08:43:32.876 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.rate_limiter: (nil) 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.unordered_write: 0 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.row_cache: None 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.wal_filter: None 2024-09-17T08:43:32.877 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.preserve_deletes: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.two_write_queues: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.manual_wal_flush: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.atomic_flush: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.log_readahead_size: 0 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_background_jobs: 2 2024-09-17T08:43:32.878 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_background_compactions: -1 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_total_wal_size: 0 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_open_files: -1 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bytes_per_sync: 0 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T08:43:32.879 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Compression algorithms supported: 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kZSTD supported: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kXpressCompression supported: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kLZ4Compression supported: 1 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kBZip2Compression supported: 0 2024-09-17T08:43:32.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kZlibCompression supported: 1 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: kSnappyCompression supported: 1 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.merge_operator: 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_filter: None 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_filter_factory: None 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T08:43:32.881 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558a872b00d8) 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cache_index_and_filter_blocks: 1 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: pin_top_level_index_and_filter: 1 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: index_type: 0 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: data_block_index_type: 0 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: index_shortening: 1 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: data_block_hash_table_util_ratio: 0.750000 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: hash_index_allow_collision: 1 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: checksum: 1 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: no_block_cache: 0 2024-09-17T08:43:32.882 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_cache: 0x558a872e6f10 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_cache_name: BinnedLRUCache 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_cache_options: 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: capacity : 536870912 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: num_shard_bits : 4 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: strict_capacity_limit : 0 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: high_pri_pool_ratio: 0.000 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_cache_compressed: (nil) 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: persistent_cache: (nil) 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_size: 4096 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_size_deviation: 10 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_restart_interval: 16 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: index_block_restart_interval: 1 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: metadata_block_size: 4096 2024-09-17T08:43:32.883 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: partition_filters: 0 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: use_delta_encoding: 1 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: whole_key_filtering: 1 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: verify_compression: 0 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: read_amp_bytes_per_bit: 0 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: format_version: 2 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: enable_index_compression: 1 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: block_align: 0 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-17T08:43:32.884 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression: NoCompression 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.num_levels: 7 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:43:32.885 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.level: 32767 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compression_opts.enabled: false 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T08:43:32.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.arena_block_size: 4194304 2024-09-17T08:43:32.887 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.526+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T08:43:32.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.table_properties_collectors: 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.inplace_update_support: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.bloom_locality: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.max_successive_merges: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.force_consistency_checks: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.ttl: 2592000 2024-09-17T08:43:32.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562612529120, "job": 1, "event": "recovery_started", "log_files": [6]} 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.527+0000 7fb430265700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2024-09-17T08:43:32.890 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.528+0000 7fb430265700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.528+0000 7fb430265700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562612530037, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83430, "table_properties": {"data_size": 81658, "index_size": 377, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9060, "raw_average_key_size": 45, "raw_value_size": 76331, "raw_average_value_size": 383, "num_data_blocks": 12, "num_entries": 199, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726562612, "oldest_key_time": 3, "file_creation_time": 0}} 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.528+0000 7fb430265700 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.529+0000 7fb430265700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562612530449, "job": 1, "event": "recovery_finished"} 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.529+0000 7fb430265700 4 rocksdb: DB pointer 0x558a873ab800 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.529+0000 7fb41707f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.529+0000 7fb41707f700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** DB Stats ** 2024-09-17T08:43:32.891 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** Compaction Stats [default] ** 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: L0 2/0 83.13 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Sum 2/0 83.13 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** Compaction Stats [default] ** 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:43:32.893 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Cumulative compaction: 0.00 GB write, 27.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Interval compaction: 0.00 GB write, 27.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** Compaction Stats [default] ** 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: L0 2/0 83.13 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Sum 2/0 83.13 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:43:32.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** Compaction Stats [default] ** 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 137.2 0.00 0.00 1 0.001 0 0 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Cumulative compaction: 0.00 GB write, 26.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:32.895 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.530+0000 7fb430265700 0 starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 1 mon.a@-1(???) e1 preinit fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).mds e1 new map 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).mds e1 print_map 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: e1 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T08:43:32.896 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: legacy client fscid: -1 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: No filesystems configured 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:30.927029+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.27:0/2618393449' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T08:43:32.531+0000 7fb430265700 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:32.536371+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:32.536429+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:32.536461+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-09-17T08:43:32.897 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:32.536485+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-17T08:43:32.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:32.536816+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-09-17T08:43:32.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 bash[23594]: ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 2024-09-17T08:43:32.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:32 smithi027 systemd[1]: Started Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:43:33.658 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T08:43:33.658 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T08:43:33.659 INFO:teuthology.orchestra.run.smithi027.stderr:Creating mgr... 2024-09-17T08:43:33.659 INFO:teuthology.orchestra.run.smithi027.stderr:Verifying port 9283 ... 2024-09-17T08:43:33.661 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl daemon-reload 2024-09-17T08:43:33.811 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y 2024-09-17T08:43:33.826 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl reset-failed ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y 2024-09-17T08:43:33.832 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Failed to reset failed state of unit ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-17T08:43:33.833 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl enable ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y 2024-09-17T08:43:33.840 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: Created symlink /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968.target.wants/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@.service. 2024-09-17T08:43:33.982 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl start ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y 2024-09-17T08:43:34.170 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 systemd[1]: Starting Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:43:34.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:33.170507+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.27:0/810523359' entity='client.admin' 2024-09-17T08:43:34.500 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 podman[23805]: 2024-09-17 08:43:34.0745541 +0000 UTC m=+0.015971003 image pull 24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 quay.io/ceph/ceph:v16.2.0 2024-09-17T08:43:34.500 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 podman[23805]: 2024-09-17 08:43:34.172675216 +0000 UTC m=+0.114092122 container create 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.build-date=20201204, RELEASE=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, maintainer=Dimitri Savineau , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T08:43:34.500 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 podman[23805]: 2024-09-17 08:43:34.399855604 +0000 UTC m=+0.341272507 container init 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, ceph=True, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20201204, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, maintainer=Dimitri Savineau ) 2024-09-17T08:43:34.500 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 podman[23805]: 2024-09-17 08:43:34.40272912 +0000 UTC m=+0.344146017 container start 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, org.label-schema.schema-version=1.0, org.label-schema.build-date=20201204, org.label-schema.vendor=CentOS, maintainer=Dimitri Savineau , GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T08:43:34.504 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled firewalld.service 2024-09-17T08:43:34.513 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-09-17T08:43:34.513 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active firewalld.service 2024-09-17T08:43:34.520 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:43:34.521 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-09-17T08:43:34.521 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T08:43:34.522 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[]>. firewalld.service is not available 2024-09-17T08:43:34.522 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled firewalld.service 2024-09-17T08:43:34.529 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-09-17T08:43:34.530 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active firewalld.service 2024-09-17T08:43:34.537 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:43:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-09-17T08:43:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T08:43:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr to start... 2024-09-17T08:43:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr... 2024-09-17T08:43:34.538 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-09-17T08:43:34.758 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 bash[23805]: 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 2024-09-17T08:43:34.758 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:34 smithi027 systemd[1]: Started Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "d61dd7c6-74d0-11ef-bceb-c7b262605968", 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.290 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.291 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T08:43:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T08:43:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T08:43:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T08:43:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T08:43:36.295 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.296 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": false, 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-09-17T08:43:19.990005+0000", 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:36.297 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:36.298 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T08:43:36.298 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:43:36.632 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:36.286370+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.27:0/1931491497' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T08:43:36.959 INFO:teuthology.orchestra.run.smithi027.stderr:mgr not available, waiting (1/15)... 2024-09-17T08:43:38.962 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "d61dd7c6-74d0-11ef-bceb-c7b262605968", 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T08:43:39.456 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:39.457 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T08:43:39.458 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T08:43:39.458 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T08:43:39.458 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T08:43:39.461 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": false, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-09-17T08:43:39.462 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-09-17T08:43:19.990005+0000", 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T08:43:39.463 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:43:39.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.453429+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.27:0/2415630216' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T08:43:40.016 INFO:teuthology.orchestra.run.smithi027.stderr:mgr not available, waiting (2/15)... 2024-09-17T08:43:40.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:39.549769+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:39.553162+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00353075s) 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.554173+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.554390+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.554593+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.554785+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.554976+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:39.556966+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.565508+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.569197+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.570467+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:43:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.576028+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' 2024-09-17T08:43:40.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:39.578371+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.27:0/3908086480' entity='mgr.y' 2024-09-17T08:43:42.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:40.555526+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.00589s) 2024-09-17T08:43:42.018 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 status -f json-pretty 2024-09-17T08:43:42.702 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:41.557088+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e4: y(active, since 2s) 2024-09-17T08:43:42.707 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 2024-09-17T08:43:42.707 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:43:42.707 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsid": "d61dd7c6-74d0-11ef-bceb-c7b262605968", 2024-09-17T08:43:42.707 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "health": { 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 0 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "a" 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "quorum_age": 10, 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T08:43:42.708 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.709 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.710 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "iostat", 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "restful" 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ], 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "modified": "2024-09-17T08:43:19.990005+0000", 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "services": {} 2024-09-17T08:43:42.711 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: }, 2024-09-17T08:43:42.712 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T08:43:42.712 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:43:43.281 INFO:teuthology.orchestra.run.smithi027.stderr:mgr is available 2024-09-17T08:43:43.281 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config-key set mgr/cephadm/ssh_user root 2024-09-17T08:43:43.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:42.702276+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.27:0/1662596473' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T08:43:43.943 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: set mgr/cephadm/ssh_user 2024-09-17T08:43:44.478 INFO:teuthology.orchestra.run.smithi027.stderr:Enabling cephadm module... 2024-09-17T08:43:44.479 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr module enable cephadm 2024-09-17T08:43:45.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:43.941485+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.27:0/1120722177' entity='client.admin' 2024-09-17T08:43:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:45.027592+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.27:0/2789553395' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T08:43:46.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setuser ceph since I am not root 2024-09-17T08:43:46.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setgroup ceph since I am not root 2024-09-17T08:43:46.543 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr stat 2024-09-17T08:43:47.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:45.946099+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.27:0/2789553395' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T08:43:47.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:45.946153+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2024-09-17T08:43:47.174 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:43:47.175 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T08:43:47.175 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-09-17T08:43:47.175 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-17T08:43:47.175 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T08:43:47.175 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:43:47.774 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for the mgr to restart... 2024-09-17T08:43:47.774 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr epoch 5... 2024-09-17T08:43:47.774 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 tell mgr mgr_status 2024-09-17T08:43:48.082 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:47.174071+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.27:0/3231602597' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T08:43:51.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:51.105097+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2024-09-17T08:43:51.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:51.105481+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2024-09-17T08:43:51.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:51.107613+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-09-17T08:43:52.174 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:43:52.174 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T08:43:52.174 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T08:43:52.174 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:51.158565+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.0531663s) 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.160916+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.161655+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.162445+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.162673+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.162878+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:51.164595+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.177858+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.180483+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.182891+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.278788+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.279481+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.280600+0000 mon.a (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.281929+0000 mon.a (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.292672+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:43:52.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:51.293909+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:43:52.714 INFO:teuthology.orchestra.run.smithi027.stderr:mgr epoch 5 is available 2024-09-17T08:43:52.714 INFO:teuthology.orchestra.run.smithi027.stderr:Setting orchestrator backend to cephadm... 2024-09-17T08:43:52.714 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch set backend cephadm 2024-09-17T08:43:53.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:52.166080+0000 mgr.y (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T08:43:53.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:52.166812+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e7: y(active, since 1.0614s) 2024-09-17T08:43:53.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:52.173370+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T08:43:53.878 INFO:teuthology.orchestra.run.smithi027.stderr:Generating ssh key... 2024-09-17T08:43:53.878 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 cephadm generate-key 2024-09-17T08:43:54.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:53.318477+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:43:54.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:53.322272+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:54.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:53.325458+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:43:54.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:43:53.541477+0000 mon.a (mon.0) 52 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-09-17T08:43:54.989 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Generating public/private rsa key pair. 2024-09-17T08:43:54.989 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Your identification has been saved in /tmp/tmpvdpn7grn/key. 2024-09-17T08:43:54.989 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Your public key has been saved in /tmp/tmpvdpn7grn/key.pub. 2024-09-17T08:43:54.989 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: The key fingerprint is: 2024-09-17T08:43:54.989 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: SHA256:Ha0UvZDYPl3vCipy3EER4XINDGkTNIkLCI9gfyUOhDY ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: The key's randomart image is: 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: +---[RSA 3072]----+ 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: |ooo+ . +=O== | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: |oEo + + *o*=. . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: |o o. + o.o=+oo . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | . . =+oo .| 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | S.o. . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | . . .| 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | . . o . . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | . + o . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: | o . | 2024-09-17T08:43:54.990 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: +----[SHA256]-----+ 2024-09-17T08:43:54.991 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 cephadm get-pub-key 2024-09-17T08:43:55.578 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:43:54.414315+0000 mgr.y (mgr.14116) 5 : cephadm [INF] Generating ssh key... 2024-09-17T08:43:55.578 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:54.532731+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:55.579 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:54.535114+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:55.579 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLH7tuvWjiB3s+m2VgdCck/OgNINnQUeQdXoRJYHlUbU17n83elszUXNA3ZptOiTmN0n403pPuOmB6o3nLk+zuPmoDsA1xNOko0j4YegNKKi+YNaesCERcf9YTKx26tF+1cBjYoUkixWLpPW1PL6CLz0dERAyj6KUWjf3vcl4EPjO3yWD05j2RQtvXZPxx2saWzqXgZR6FzqNfF2iwsKzRA+fc0DXpQoHypWhkvMRtyDl461n3254eJBEFhpC6oh63vewwgsUC0y4QNvJWqJHZiJy0NQsQBc/2/Ie+bbrFoEqycrhMvR6nLS7qdRL72OVY0vZlgt+dLufrbR+LVLN1KlgrHEdmW6c8tAeOdQHQY/Izt/vSqQFqN9a9JNyzb/4TZIdtAlGPlg5k3V8NoYvl2fHj0zLRx9cHCd7LPjO3c9dLKghLAcbbfCamjQsTkEYOFdvnSnKlZBVp5wZO/73iqw7j6Hc4DOcD6n9vp3QDbcgWq4OZgcsDFhKm/O25gTk= ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:43:56.125 INFO:teuthology.orchestra.run.smithi027.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T08:43:56.125 INFO:teuthology.orchestra.run.smithi027.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T08:43:56.135 INFO:teuthology.orchestra.run.smithi027.stderr:Adding host smithi027... 2024-09-17T08:43:56.135 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch host add smithi027 2024-09-17T08:43:56.742 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:55.578676+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:43:57.007 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:43:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Warning: Permanently added 'smithi027,172.21.15.27' (ECDSA) to the list of known hosts. 2024-09-17T08:43:57.333 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Added host 'smithi027' 2024-09-17T08:43:57.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:56.716973+0000 mgr.y (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:43:57.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:57.331929+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:43:57.643 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:57.332464+0000 mon.a (mon.0) 56 : audit [DBG] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:43:57.921 INFO:teuthology.orchestra.run.smithi027.stderr:Deploying unmanaged mon service... 2024-09-17T08:43:57.921 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply mon --unmanaged 2024-09-17T08:43:58.631 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:43:57.332231+0000 mgr.y (mgr.14116) 8 : cephadm [INF] Added host smithi027 2024-09-17T08:43:58.792 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T08:43:59.906 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:58.788548+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:43:59.907 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:43:58.789275+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Saving service mon spec with placement count:5 2024-09-17T08:43:59.907 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:43:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:58.791607+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:44:00.277 INFO:teuthology.orchestra.run.smithi027.stderr:Deploying unmanaged mgr service... 2024-09-17T08:44:00.277 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 orch apply mgr --unmanaged 2024-09-17T08:44:00.880 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T08:44:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:43:59.933601+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:44:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:00.140277+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:44:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:00.880204+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:44:01.560 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mgr mgr/cephadm/container_init True --force 2024-09-17T08:44:02.102 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:00.877647+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:02.102 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:00.878149+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mgr spec with placement count:2 2024-09-17T08:44:02.910 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config set mgr mgr/dashboard/ssl_server_port 8443 2024-09-17T08:44:03.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:02.296524+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.27:0/2701051056' entity='client.admin' 2024-09-17T08:44:04.091 INFO:teuthology.orchestra.run.smithi027.stderr:Enabling the dashboard module... 2024-09-17T08:44:04.091 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr module enable dashboard 2024-09-17T08:44:04.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:03.404541+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.14116 172.21.15.27:0/1395227975' entity='mgr.y' 2024-09-17T08:44:04.500 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:03.517339+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.27:0/3193996073' entity='client.admin' 2024-09-17T08:44:05.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:05.455720+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.27:0/3269329431' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T08:44:05.942 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:44:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setuser ceph since I am not root 2024-09-17T08:44:05.942 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:44:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setgroup ceph since I am not root 2024-09-17T08:44:06.254 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 mgr stat 2024-09-17T08:44:06.963 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:05.509144+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.27:0/3269329431' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T08:44:06.963 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:05.509219+0000 mon.a (mon.0) 66 : cluster [DBG] mgrmap e9: y(active, since 14s) 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "available": true, 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "active_name": "y", 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T08:44:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:44:07.677 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for the mgr to restart... 2024-09-17T08:44:07.677 INFO:teuthology.orchestra.run.smithi027.stderr:Waiting for mgr epoch 9... 2024-09-17T08:44:07.678 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 tell mgr mgr_status 2024-09-17T08:44:07.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:06.963606+0000 mon.a (mon.0) 67 : audit [DBG] from='client.? 172.21.15.27:0/2587267142' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T08:44:10.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:10.699538+0000 mon.a (mon.0) 68 : cluster [INF] Active manager daemon y restarted 2024-09-17T08:44:10.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:10.699913+0000 mon.a (mon.0) 69 : cluster [INF] Activating manager daemon y 2024-09-17T08:44:10.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:10.701856+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-09-17T08:44:11.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:44:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Warning: Permanently added 'smithi027,172.21.15.27' (ECDSA) to the list of known hosts. 2024-09-17T08:44:11.764 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: { 2024-09-17T08:44:11.764 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T08:44:11.764 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T08:44:11.764 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: } 2024-09-17T08:44:12.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:10.752330+0000 mon.a (mon.0) 71 : cluster [DBG] mgrmap e10: y(active, starting, since 0.0525006s) 2024-09-17T08:44:12.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.754559+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.755264+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.755990+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.756120+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.756235+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:10.758072+0000 mon.a (mon.0) 77 : cluster [INF] Manager daemon y is now available 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.865202+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.867321+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.879843+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:44:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:10.881077+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:44:12.697 INFO:teuthology.orchestra.run.smithi027.stderr:mgr epoch 9 is available 2024-09-17T08:44:12.698 INFO:teuthology.orchestra.run.smithi027.stderr:Generating a dashboard self-signed certificate... 2024-09-17T08:44:12.698 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 dashboard create-self-signed-cert 2024-09-17T08:44:12.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:11.756257+0000 mgr.y (mgr.14144) 1 : audit [DBG] from='client.14148 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T08:44:12.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:11.757009+0000 mon.a (mon.0) 82 : cluster [DBG] mgrmap e11: y(active, since 1.05717s) 2024-09-17T08:44:12.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:11.763642+0000 mgr.y (mgr.14144) 2 : audit [DBG] from='client.14148 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T08:44:12.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:12.746039+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:13.485 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T08:44:14.366 INFO:teuthology.orchestra.run.smithi027.stderr:Creating initial admin user... 2024-09-17T08:44:14.367 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmp25_f_z9j:/tmp/dashboard.pw:z quay.io/ceph/ceph:v16.2.0 dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2024-09-17T08:44:14.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:13.360090+0000 mgr.y (mgr.14144) 3 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:14.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:13.482812+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:14.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:13.484715+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:14.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:13.544664+0000 mon.a (mon.0) 86 : cluster [DBG] mgrmap e12: y(active, since 2s) 2024-09-17T08:44:15.241 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Zz7oPSbp.nxe2Y4kO7/rJud6r74e.UAzgLOfP9eoETT//NPJ9fu9W", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726562655, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T08:44:15.807 INFO:teuthology.orchestra.run.smithi027.stderr:Fetching dashboard port number... 2024-09-17T08:44:15.808 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v16.2.0 -e NODE_NAME=smithi027 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968:/var/log/ceph:z -v /tmp/ceph-tmp9l5l1mts:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmprn_umnf_:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v16.2.0 config get mgr mgr/dashboard/ssl_server_port 2024-09-17T08:44:16.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:14.995830+0000 mgr.y (mgr.14144) 4 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:16.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:15.240675+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:16.425 INFO:teuthology.orchestra.run.smithi027.stderr:/usr/bin/ceph: 8443 2024-09-17T08:44:17.031 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-enabled firewalld.service 2024-09-17T08:44:17.040 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: disabled 2024-09-17T08:44:17.041 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: systemctl is-active firewalld.service 2024-09-17T08:44:17.047 INFO:teuthology.orchestra.run.smithi027.stderr:systemctl: inactive 2024-09-17T08:44:17.048 INFO:teuthology.orchestra.run.smithi027.stderr:firewalld.service is not enabled 2024-09-17T08:44:17.048 INFO:teuthology.orchestra.run.smithi027.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T08:44:17.051 INFO:teuthology.orchestra.run.smithi027.stderr:Ceph Dashboard is now available at: 2024-09-17T08:44:17.051 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.051 INFO:teuthology.orchestra.run.smithi027.stderr: URL: https://smithi027.front.sepia.ceph.com:8443/ 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: User: admin 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: Password: iolc6qa6um 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr:You can access the Ceph CLI with: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: ceph telemetry on 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr:For more information see: 2024-09-17T08:44:17.052 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.053 INFO:teuthology.orchestra.run.smithi027.stderr: https://docs.ceph.com/docs/pacific/mgr/telemetry/ 2024-09-17T08:44:17.053 INFO:teuthology.orchestra.run.smithi027.stderr: 2024-09-17T08:44:17.053 INFO:teuthology.orchestra.run.smithi027.stderr:Bootstrap complete. 2024-09-17T08:44:17.053 INFO:teuthology.orchestra.run.smithi027.stderr:Releasing lock 140053671299104 on /run/cephadm/d61dd7c6-74d0-11ef-bceb-c7b262605968.lock 2024-09-17T08:44:17.053 INFO:teuthology.orchestra.run.smithi027.stderr:Lock 140053671299104 released on /run/cephadm/d61dd7c6-74d0-11ef-bceb-c7b262605968.lock 2024-09-17T08:44:17.085 INFO:tasks.cephadm:Fetching config... 2024-09-17T08:44:17.086 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:17.086 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T08:44:17.103 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T08:44:17.103 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:17.103 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T08:44:17.160 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T08:44:17.160 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:17.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-17T08:44:17.225 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T08:44:17.226 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:17.226 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T08:44:17.282 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T08:44:17.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLH7tuvWjiB3s+m2VgdCck/OgNINnQUeQdXoRJYHlUbU17n83elszUXNA3ZptOiTmN0n403pPuOmB6o3nLk+zuPmoDsA1xNOko0j4YegNKKi+YNaesCERcf9YTKx26tF+1cBjYoUkixWLpPW1PL6CLz0dERAyj6KUWjf3vcl4EPjO3yWD05j2RQtvXZPxx2saWzqXgZR6FzqNfF2iwsKzRA+fc0DXpQoHypWhkvMRtyDl461n3254eJBEFhpC6oh63vewwgsUC0y4QNvJWqJHZiJy0NQsQBc/2/Ie+bbrFoEqycrhMvR6nLS7qdRL72OVY0vZlgt+dLufrbR+LVLN1KlgrHEdmW6c8tAeOdQHQY/Izt/vSqQFqN9a9JNyzb/4TZIdtAlGPlg5k3V8NoYvl2fHj0zLRx9cHCd7LPjO3c9dLKghLAcbbfCamjQsTkEYOFdvnSnKlZBVp5wZO/73iqw7j6Hc4DOcD6n9vp3QDbcgWq4OZgcsDFhKm/O25gTk= ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T08:44:17.359 INFO:teuthology.orchestra.run.smithi027.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLH7tuvWjiB3s+m2VgdCck/OgNINnQUeQdXoRJYHlUbU17n83elszUXNA3ZptOiTmN0n403pPuOmB6o3nLk+zuPmoDsA1xNOko0j4YegNKKi+YNaesCERcf9YTKx26tF+1cBjYoUkixWLpPW1PL6CLz0dERAyj6KUWjf3vcl4EPjO3yWD05j2RQtvXZPxx2saWzqXgZR6FzqNfF2iwsKzRA+fc0DXpQoHypWhkvMRtyDl461n3254eJBEFhpC6oh63vewwgsUC0y4QNvJWqJHZiJy0NQsQBc/2/Ie+bbrFoEqycrhMvR6nLS7qdRL72OVY0vZlgt+dLufrbR+LVLN1KlgrHEdmW6c8tAeOdQHQY/Izt/vSqQFqN9a9JNyzb/4TZIdtAlGPlg5k3V8NoYvl2fHj0zLRx9cHCd7LPjO3c9dLKghLAcbbfCamjQsTkEYOFdvnSnKlZBVp5wZO/73iqw7j6Hc4DOcD6n9vp3QDbcgWq4OZgcsDFhKm/O25gTk= ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:17.371 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLH7tuvWjiB3s+m2VgdCck/OgNINnQUeQdXoRJYHlUbU17n83elszUXNA3ZptOiTmN0n403pPuOmB6o3nLk+zuPmoDsA1xNOko0j4YegNKKi+YNaesCERcf9YTKx26tF+1cBjYoUkixWLpPW1PL6CLz0dERAyj6KUWjf3vcl4EPjO3yWD05j2RQtvXZPxx2saWzqXgZR6FzqNfF2iwsKzRA+fc0DXpQoHypWhkvMRtyDl461n3254eJBEFhpC6oh63vewwgsUC0y4QNvJWqJHZiJy0NQsQBc/2/Ie+bbrFoEqycrhMvR6nLS7qdRL72OVY0vZlgt+dLufrbR+LVLN1KlgrHEdmW6c8tAeOdQHQY/Izt/vSqQFqN9a9JNyzb/4TZIdtAlGPlg5k3V8NoYvl2fHj0zLRx9cHCd7LPjO3c9dLKghLAcbbfCamjQsTkEYOFdvnSnKlZBVp5wZO/73iqw7j6Hc4DOcD6n9vp3QDbcgWq4OZgcsDFhKm/O25gTk= ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T08:44:17.442 INFO:teuthology.orchestra.run.smithi105.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLH7tuvWjiB3s+m2VgdCck/OgNINnQUeQdXoRJYHlUbU17n83elszUXNA3ZptOiTmN0n403pPuOmB6o3nLk+zuPmoDsA1xNOko0j4YegNKKi+YNaesCERcf9YTKx26tF+1cBjYoUkixWLpPW1PL6CLz0dERAyj6KUWjf3vcl4EPjO3yWD05j2RQtvXZPxx2saWzqXgZR6FzqNfF2iwsKzRA+fc0DXpQoHypWhkvMRtyDl461n3254eJBEFhpC6oh63vewwgsUC0y4QNvJWqJHZiJy0NQsQBc/2/Ie+bbrFoEqycrhMvR6nLS7qdRL72OVY0vZlgt+dLufrbR+LVLN1KlgrHEdmW6c8tAeOdQHQY/Izt/vSqQFqN9a9JNyzb/4TZIdtAlGPlg5k3V8NoYvl2fHj0zLRx9cHCd7LPjO3c9dLKghLAcbbfCamjQsTkEYOFdvnSnKlZBVp5wZO/73iqw7j6Hc4DOcD6n9vp3QDbcgWq4OZgcsDFhKm/O25gTk= ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:17.455 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi105 2024-09-17T08:44:17.455 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:44:17.455 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.conf 2024-09-17T08:44:17.472 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:44:17.472 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T08:44:17.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:16.425212+0000 mon.a (mon.0) 88 : audit [DBG] from='client.? 172.21.15.27:0/485376232' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T08:44:17.528 INFO:tasks.cephadm:Adding host smithi105 to orchestrator... 2024-09-17T08:44:17.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch host add smithi105 2024-09-17T08:44:18.494 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:17.243918+0000 mon.a (mon.0) 89 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-09-17T08:44:20.085 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:44:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: Warning: Permanently added 'smithi105,172.21.15.105' (ECDSA) to the list of known hosts. 2024-09-17T08:44:20.488 INFO:teuthology.orchestra.run.smithi027.stdout:Added host 'smithi105' 2024-09-17T08:44:21.142 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T08:44:21.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.153574+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:21.177 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.438341+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.439902+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.443982+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.487193+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:20.487508+0000 mgr.y (mgr.14144) 6 : cephadm [INF] Added host smithi105 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.487719+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.687559+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:21.178 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:20.880602+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:23.601 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:44:23.601 INFO:teuthology.orchestra.run.smithi027.stdout:[{"addr": "smithi027", "hostname": "smithi027", "labels": [], "status": ""}, {"addr": "smithi105", "hostname": "smithi105", "labels": [], "status": ""}] 2024-09-17T08:44:24.176 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T08:44:24.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T08:44:25.048 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:23.601100+0000 mgr.y (mgr.14144) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:44:25.049 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:23.780515+0000 mon.a (mon.0) 98 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:25.049 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:24.016527+0000 mon.a (mon.0) 99 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:25.049 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:24.018558+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:25.971 INFO:teuthology.orchestra.run.smithi027.stderr:adjusted tunables profile to default 2024-09-17T08:44:26.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:25.916098+0000 mon.a (mon.0) 101 : audit [INF] from='client.? 172.21.15.27:0/3116267984' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T08:44:26.496 INFO:tasks.cephadm:Adding mon.a on smithi027 2024-09-17T08:44:26.496 INFO:tasks.cephadm:Adding mon.c on smithi027 2024-09-17T08:44:26.496 INFO:tasks.cephadm:Adding mon.b on smithi105 2024-09-17T08:44:26.496 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi105:172.21.15.105=b' 2024-09-17T08:44:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:25.968605+0000 mon.a (mon.0) 102 : audit [INF] from='client.? 172.21.15.27:0/3116267984' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T08:44:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:25.968648+0000 mon.a (mon.0) 103 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:28.312 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mon update... 2024-09-17T08:44:28.932 DEBUG:teuthology.orchestra.run.smithi027:mon.c> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.c.service 2024-09-17T08:44:28.935 DEBUG:teuthology.orchestra.run.smithi105:mon.b> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.b.service 2024-09-17T08:44:28.940 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T08:44:28.941 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.308471+0000 mgr.y (mgr.14144) 8 : audit [DBG] from='client.14166 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi105:172.21.15.105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:28.309686+0000 mgr.y (mgr.14144) 9 : cephadm [INF] Saving service mon spec with placement smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.312014+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.312554+0000 mon.a (mon.0) 105 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.318624+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.319613+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:29.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:28.320271+0000 mon.a (mon.0) 108 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:29.571 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:28.320896+0000 mgr.y (mgr.14144) 10 : cephadm [INF] Deploying daemon mon.c on smithi027 2024-09-17T08:44:30.613 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:44:30.613 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":1,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","modified":"2024-09-17T08:43:16.507114Z","created":"2024-09-17T08:43:16.507114Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T08:44:30.614 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 1 2024-09-17T08:44:31.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:30.613198+0000 mon.a (mon.0) 109 : audit [DBG] from='client.? 172.21.15.105:0/1580963512' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:31.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:31 smithi027 systemd[1]: Started Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:44:32.161 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T08:44:32.161 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T08:44:36.849 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:44:36.849 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":2,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","modified":"2024-09-17T08:44:31.520199Z","created":"2024-09-17T08:43:16.507114Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T08:44:36.851 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 2 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:30.757344+0000 mgr.y (mgr.14144) 11 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:31.522088+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:31.522224+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:31.522325+0000 mon.a (mon.0) 113 : cluster [INF] mon.a calling monitor election 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:32.519847+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:32.757718+0000 mgr.y (mgr.14144) 12 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:33.520032+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:33.521316+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:34.520078+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:34.757996+0000 mgr.y (mgr.14144) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:35.520303+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:36.520549+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.525036+0000 mon.a (mon.0) 119 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.527948+0000 mon.a (mon.0) 120 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.528009+0000 mon.a (mon.0) 121 : cluster [DBG] fsmap 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.528054+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.528274+0000 mon.a (mon.0) 123 : cluster [DBG] mgrmap e13: y(active, since 25s) 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.531937+0000 mon.a (mon.0) 124 : cluster [INF] overall HEALTH_OK 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:30.757344+0000 mgr.y (mgr.14144) 11 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:31.522088+0000 mon.a (mon.0) 111 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:31.522224+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:31.522325+0000 mon.a (mon.0) 113 : cluster [INF] mon.a calling monitor election 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:32.519847+0000 mon.a (mon.0) 114 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:32.757718+0000 mgr.y (mgr.14144) 12 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:33.520032+0000 mon.a (mon.0) 115 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:33.521316+0000 mon.c (mon.1) 1 : cluster [INF] mon.c calling monitor election 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:34.520078+0000 mon.a (mon.0) 116 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:34.757996+0000 mgr.y (mgr.14144) 13 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:35.520303+0000 mon.a (mon.0) 117 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:36.520549+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.525036+0000 mon.a (mon.0) 119 : cluster [INF] mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.527948+0000 mon.a (mon.0) 120 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.528009+0000 mon.a (mon.0) 121 : cluster [DBG] fsmap 2024-09-17T08:44:37.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.528054+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.528274+0000 mon.a (mon.0) 123 : cluster [DBG] mgrmap e13: y(active, since 25s) 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.531937+0000 mon.a (mon.0) 124 : cluster [INF] overall HEALTH_OK 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:36.533996+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:36.534820+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:36.535635+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:37.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:36.536291+0000 mgr.y (mgr.14144) 14 : cephadm [INF] Deploying daemon mon.b on smithi105 2024-09-17T08:44:37.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:36.533996+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:37.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:36.534820+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:37.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:36.535635+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:37.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:36.536291+0000 mgr.y (mgr.14144) 14 : cephadm [INF] Deploying daemon mon.b on smithi105 2024-09-17T08:44:38.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:36.758333+0000 mgr.y (mgr.14144) 15 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:38.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:36.849541+0000 mon.a (mon.0) 128 : audit [DBG] from='client.? 172.21.15.105:0/1356191551' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:38.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:37.520690+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:36.758333+0000 mgr.y (mgr.14144) 15 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:36.849541+0000 mon.a (mon.0) 128 : audit [DBG] from='client.? 172.21.15.105:0/1356191551' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:37.520690+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:40.208 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-17T08:44:40.208 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T08:44:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:38.758752+0000 mgr.y (mgr.14144) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:38.758752+0000 mgr.y (mgr.14144) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:40.508 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:40 smithi105 systemd[1]: Starting Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:44:41.700 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 podman[23822]: 2024-09-17 08:44:41.397127297 +0000 UTC m=+0.739952693 container create 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.vendor=CentOS, org.label-schema.build-date=20201204, RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T08:44:41.700 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 podman[23822]: 2024-09-17 08:44:41.303526892 +0000 UTC m=+0.646352256 image pull 24ecd6d5f14c2268c5d2c666aa151456949cf76c6747d873c6fcf41f6f672d15 quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000 2024-09-17T08:44:41.700 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 podman[23822]: 2024-09-17 08:44:41.700335725 +0000 UTC m=+1.043161080 container init 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, RELEASE=HEAD, ceph=True, org.label-schema.name=CentOS Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-16.2.0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2) 2024-09-17T08:44:42.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 podman[23822]: 2024-09-17 08:44:41.703603055 +0000 UTC m=+1.046428401 container start 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD, ceph=True, maintainer=Dimitri Savineau ) 2024-09-17T08:44:42.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.722+0000 7f3b3fc53700 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-17T08:44:42.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.722+0000 7f3b3fc53700 0 ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable), process ceph-mon, pid 2 2024-09-17T08:44:42.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.722+0000 7f3b3fc53700 0 pidfile_write: ignore empty --pid-file 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 0 load: jerasure load: lrc load: isa 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: RocksDB version: 6.8.1 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Compile date Mar 30 2021 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: DB SUMMARY 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: CURRENT file: CURRENT 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: IDENTITY file: IDENTITY 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2024-09-17T08:44:42.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 636 ; 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.error_if_exists: 0 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.create_if_missing: 0 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.paranoid_checks: 1 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.env: 0x55838b39d1c0 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.fs: Posix File System 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.info_log: 0x55838d23c640 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.statistics: (nil) 2024-09-17T08:44:42.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.use_fsync: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_log_file_size: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.allow_fallocate: 1 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.use_direct_reads: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.create_missing_column_families: 0 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.db_log_dir: 2024-09-17T08:44:42.022 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_subcompactions: 1 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_background_flushes: -1 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.advise_random_on_open: 1 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-17T08:44:42.024 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.write_buffer_manager: 0x55838d2d3e30 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.rate_limiter: (nil) 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.unordered_write: 0 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T08:44:42.025 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.row_cache: None 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.wal_filter: None 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.preserve_deletes: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.two_write_queues: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.manual_wal_flush: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.atomic_flush: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T08:44:42.026 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.log_readahead_size: 0 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_background_jobs: 2 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_background_compactions: -1 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_total_wal_size: 0 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.max_open_files: -1 2024-09-17T08:44:42.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.bytes_per_sync: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Compression algorithms supported: 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kZSTD supported: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kXpressCompression supported: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kLZ4Compression supported: 1 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kBZip2Compression supported: 0 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kZlibCompression supported: 1 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: kSnappyCompression supported: 1 2024-09-17T08:44:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.727+0000 7f3b3fc53700 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.merge_operator: 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_filter: None 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_filter_factory: None 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55838d1e20d8) 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cache_index_and_filter_blocks: 1 2024-09-17T08:44:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: pin_top_level_index_and_filter: 1 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: index_type: 0 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: data_block_index_type: 0 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: index_shortening: 1 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: data_block_hash_table_util_ratio: 0.750000 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: hash_index_allow_collision: 1 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: checksum: 1 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: no_block_cache: 0 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_cache: 0x55838d218f10 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_cache_name: BinnedLRUCache 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_cache_options: 2024-09-17T08:44:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: capacity : 536870912 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: num_shard_bits : 4 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: strict_capacity_limit : 0 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: high_pri_pool_ratio: 0.000 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_cache_compressed: (nil) 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: persistent_cache: (nil) 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_size: 4096 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_size_deviation: 10 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_restart_interval: 16 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: index_block_restart_interval: 1 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: metadata_block_size: 4096 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: partition_filters: 0 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: use_delta_encoding: 1 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T08:44:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: whole_key_filtering: 1 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: verify_compression: 0 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: read_amp_bytes_per_bit: 0 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: format_version: 2 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: enable_index_compression: 1 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: block_align: 0 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression: NoCompression 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T08:44:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.num_levels: 7 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.level: 32767 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T08:44:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compression_opts.enabled: false 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T08:44:42.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T08:44:42.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.arena_block_size: 4194304 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T08:44:42.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.table_properties_collectors: 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.inplace_update_support: 0 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T08:44:42.037 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.bloom_locality: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.max_successive_merges: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.force_consistency_checks: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.ttl: 2592000 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.038 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562681729723, "job": 1, "event": "recovery_started", "log_files": [3]} 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.728+0000 7f3b3fc53700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.729+0000 7f3b3fc53700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.729+0000 7f3b3fc53700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562681730197, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1562, "table_properties": {"data_size": 648, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726562681, "oldest_key_time": 3, "file_creation_time": 0}} 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.729+0000 7f3b3fc53700 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.729+0000 7f3b3fc53700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726562681730562, "job": 1, "event": "recovery_finished"} 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.730+0000 7f3b3fc53700 4 rocksdb: DB pointer 0x55838d2dd800 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.730+0000 7f3b26a6d700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.730+0000 7f3b26a6d700 4 rocksdb: [db_impl/db_impl.cc:851] 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** DB Stats ** 2024-09-17T08:44:42.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** Compaction Stats [default] ** 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.040 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** Compaction Stats [default] ** 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:44:42.041 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** Compaction Stats [default] ** 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T08:44:42.042 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** Compaction Stats [default] ** 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.5 0.00 0.00 1 0.000 0 0 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T08:44:42.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: AddFile(Keys): cumulative 0, interval 0 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ** File Read Latency Histogram By Level [default] ** 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.730+0000 7f3b3fc53700 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.730+0000 7f3b3fc53700 0 using public_addr v2:172.21.15.105:0/0 -> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.731+0000 7f3b3fc53700 0 starting mon.b rank -1 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.732+0000 7f3b3fc53700 1 mon.b@-1(???) e0 preinit fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:42.044 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).mds e1 new map 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).mds e1 print_map 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: e1 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: legacy client fscid: -1 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: No filesystems configured 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-17T08:44:42.045 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:38.758752+0000 mgr.y (mgr.14144) 16 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T08:44:41.746+0000 7f3b29272700 1 mon.b@-1(synchronizing).paxosservice(auth 1..5) refresh upgraded, format 0 -> 3 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 bash[23822]: 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 2024-09-17T08:44:42.046 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:41 smithi105 systemd[1]: Started Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:44:47.213 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:41.753838+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:47.213 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:41.754023+0000 mon.a (mon.0) 132 : cluster [INF] mon.a calling monitor election 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:41.755256+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:41.755521+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:41.755528+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:42.750698+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:42.759348+0000 mgr.y (mgr.14144) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:43.750790+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:43.752000+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:44.751187+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:44.759755+0000 mgr.y (mgr.14144) 19 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:45.751134+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:46.751392+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.757013+0000 mon.a (mon.0) 140 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.762181+0000 mon.a (mon.0) 141 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.762225+0000 mon.a (mon.0) 142 : cluster [DBG] fsmap 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.762252+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.762458+0000 mon.a (mon.0) 144 : cluster [DBG] mgrmap e13: y(active, since 36s) 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.767845+0000 mon.a (mon.0) 145 : cluster [INF] overall HEALTH_OK 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:46.773718+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:47.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:46.774427+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:41.753838+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:41.754023+0000 mon.a (mon.0) 132 : cluster [INF] mon.a calling monitor election 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:41.755256+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:41.755521+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:41.755528+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:42.750698+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:42.759348+0000 mgr.y (mgr.14144) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:43.750790+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:43.752000+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:44.751187+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:44.759755+0000 mgr.y (mgr.14144) 19 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:45.751134+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:46.751392+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.757013+0000 mon.a (mon.0) 140 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.762181+0000 mon.a (mon.0) 141 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.762225+0000 mon.a (mon.0) 142 : cluster [DBG] fsmap 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.762252+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:41.753838+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:41.754023+0000 mon.a (mon.0) 132 : cluster [INF] mon.a calling monitor election 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:41.755256+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:41.755521+0000 mon.c (mon.1) 2 : cluster [INF] mon.c calling monitor election 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:41.755528+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:42.750698+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:42.759348+0000 mgr.y (mgr.14144) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:43.750790+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:43.752000+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:44.751187+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:44.759755+0000 mgr.y (mgr.14144) 19 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:45.751134+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:46.751392+0000 mon.a (mon.0) 139 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.757013+0000 mon.a (mon.0) 140 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.762181+0000 mon.a (mon.0) 141 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.762225+0000 mon.a (mon.0) 142 : cluster [DBG] fsmap 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.762252+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.762458+0000 mon.a (mon.0) 144 : cluster [DBG] mgrmap e13: y(active, since 36s) 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.767845+0000 mon.a (mon.0) 145 : cluster [INF] overall HEALTH_OK 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:46.773718+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:46.774427+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.762458+0000 mon.a (mon.0) 144 : cluster [DBG] mgrmap e13: y(active, since 36s) 2024-09-17T08:44:47.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.767845+0000 mon.a (mon.0) 145 : cluster [INF] overall HEALTH_OK 2024-09-17T08:44:47.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:46.773718+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:47.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:46.774427+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:48.331 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-17T08:44:48.331 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":3,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","modified":"2024-09-17T08:44:41.750781Z","created":"2024-09-17T08:43:16.507114Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3301","nonce":0},{"type":"v1","addr":"172.21.15.27:6790","nonce":0}]},"addr":"172.21.15.27:6790/0","public_addr":"172.21.15.27:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-17T08:44:48.333 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 3 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:46.759997+0000 mgr.y (mgr.14144) 20 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.565614+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.748617+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.750958+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.754045+0000 mon.a (mon.0) 151 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.754913+0000 mon.a (mon.0) 152 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.755468+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:47.756068+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:48.633 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:48.330838+0000 mon.a (mon.0) 155 : audit [DBG] from='client.? 172.21.15.105:0/490875894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:48.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:46.759997+0000 mgr.y (mgr.14144) 20 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:48.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.565614+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.748617+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.750958+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:48.647 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.754045+0000 mon.a (mon.0) 151 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.754913+0000 mon.a (mon.0) 152 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.755468+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:47.756068+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:48.330838+0000 mon.a (mon.0) 155 : audit [DBG] from='client.? 172.21.15.105:0/490875894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:46.759997+0000 mgr.y (mgr.14144) 20 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:48.648 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.565614+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.748617+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.750958+0000 mon.a (mon.0) 150 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.754045+0000 mon.a (mon.0) 151 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.754913+0000 mon.a (mon.0) 152 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.755468+0000 mon.a (mon.0) 153 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:47.756068+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:48.649 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:48.330838+0000 mon.a (mon.0) 155 : audit [DBG] from='client.? 172.21.15.105:0/490875894' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T08:44:48.958 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T08:44:48.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:47.754706+0000 mgr.y (mgr.14144) 21 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:47.756567+0000 mgr.y (mgr.14144) 22 : cephadm [INF] Reconfiguring daemon mon.a on smithi027 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:48.991028+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:48.991786+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:48.992539+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:48.993318+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:49.886 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:47.754706+0000 mgr.y (mgr.14144) 21 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-17T08:44:49.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:47.756567+0000 mgr.y (mgr.14144) 22 : cephadm [INF] Reconfiguring daemon mon.a on smithi027 2024-09-17T08:44:49.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:48.991028+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:49.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:48.991786+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:49.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:48.992539+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:49.887 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:48.993318+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:50.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:47.754706+0000 mgr.y (mgr.14144) 21 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-17T08:44:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:47.756567+0000 mgr.y (mgr.14144) 22 : cephadm [INF] Reconfiguring daemon mon.a on smithi027 2024-09-17T08:44:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:48.991028+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:48.991786+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:48.992539+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:48.993318+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:50.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:48.760189+0000 mgr.y (mgr.14144) 23 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:50.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:48.991523+0000 mgr.y (mgr.14144) 24 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-17T08:44:50.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:48.994108+0000 mgr.y (mgr.14144) 25 : cephadm [INF] Reconfiguring daemon mon.c on smithi027 2024-09-17T08:44:50.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:48.760189+0000 mgr.y (mgr.14144) 23 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:50.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:48.991523+0000 mgr.y (mgr.14144) 24 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-17T08:44:50.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:48.994108+0000 mgr.y (mgr.14144) 25 : cephadm [INF] Reconfiguring daemon mon.c on smithi027 2024-09-17T08:44:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:48.760189+0000 mgr.y (mgr.14144) 23 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:48.991523+0000 mgr.y (mgr.14144) 24 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-17T08:44:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:48.994108+0000 mgr.y (mgr.14144) 25 : cephadm [INF] Reconfiguring daemon mon.c on smithi027 2024-09-17T08:44:51.766 INFO:teuthology.orchestra.run.smithi027.stdout:# minimal ceph.conf for d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:51.767 INFO:teuthology.orchestra.run.smithi027.stdout:[global] 2024-09-17T08:44:51.767 INFO:teuthology.orchestra.run.smithi027.stdout: fsid = d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:44:51.767 INFO:teuthology.orchestra.run.smithi027.stdout: mon_host = [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] 2024-09-17T08:44:52.366 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T08:44:52.367 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:52.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T08:44:52.397 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:52.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:50.760465+0000 mgr.y (mgr.14144) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:51.172711+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:51.173213+0000 mgr.y (mgr.14144) 27 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:51.173476+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:51.174193+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:51.174839+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:51.175377+0000 mgr.y (mgr.14144) 28 : cephadm [INF] Reconfiguring daemon mon.b on smithi105 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:51.766997+0000 mon.a (mon.0) 164 : audit [DBG] from='client.? 172.21.15.27:0/3488745544' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:50.760465+0000 mgr.y (mgr.14144) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:51.172711+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:51.173213+0000 mgr.y (mgr.14144) 27 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:51.173476+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:51.174193+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:51.174839+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:51.175377+0000 mgr.y (mgr.14144) 28 : cephadm [INF] Reconfiguring daemon mon.b on smithi105 2024-09-17T08:44:52.460 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:51.766997+0000 mon.a (mon.0) 164 : audit [DBG] from='client.? 172.21.15.27:0/3488745544' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.463 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:44:52.463 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:50.760465+0000 mgr.y (mgr.14144) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:51.172711+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:51.173213+0000 mgr.y (mgr.14144) 27 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:51.173476+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:51.174193+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T08:44:52.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:51.174839+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.479 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:51.175377+0000 mgr.y (mgr.14144) 28 : cephadm [INF] Reconfiguring daemon mon.b on smithi105 2024-09-17T08:44:52.479 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:51.766997+0000 mon.a (mon.0) 164 : audit [DBG] from='client.? 172.21.15.27:0/3488745544' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:52.492 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:44:52.492 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T08:44:52.558 INFO:tasks.cephadm:Adding mgr.y on smithi027 2024-09-17T08:44:52.558 INFO:tasks.cephadm:Adding mgr.x on smithi105 2024-09-17T08:44:52.558 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi027=y;smithi105=x' 2024-09-17T08:44:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:52.363944+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:52.364758+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:52.369615+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:53.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:52.363944+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:53.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:52.364758+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:53.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:52.369615+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:53.511 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:52.363944+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:53.511 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:52.364758+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:53.511 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:52.369615+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:54.444 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mgr update... 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:52.760673+0000 mgr.y (mgr.14144) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.443591+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.444305+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.449205+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.450140+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.453197+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.454042+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:54.726 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.454988+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:54.989 DEBUG:teuthology.orchestra.run.smithi105:mgr.x> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service 2024-09-17T08:44:54.991 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T08:44:54.992 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:44:54.992 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:52.760673+0000 mgr.y (mgr.14144) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.443591+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.444305+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.449205+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.450140+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:55.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.453197+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.454042+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.454988+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:52.760673+0000 mgr.y (mgr.14144) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.443591+0000 mon.a (mon.0) 168 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.444305+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.449205+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.450140+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.453197+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.454042+0000 mon.a (mon.0) 173 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.454988+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:55.012 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T08:44:55.013 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_1 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-09-17 08:44:31.008479190 +0000 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-09-17 08:42:37.466436627 +0000 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-09-17 08:42:37.466436627 +0000 2024-09-17T08:44:55.072 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-09-17 08:42:37.466436627 +0000 2024-09-17T08:44:55.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T08:44:55.141 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-09-17T08:44:55.141 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-09-17T08:44:55.141 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000287152 s, 1.8 MB/s 2024-09-17T08:44:55.142 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T08:44:55.202 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_2 2024-09-17T08:44:55.259 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-09-17 08:44:31.009479164 +0000 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-09-17 08:42:37.802427807 +0000 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-09-17 08:42:37.802427807 +0000 2024-09-17T08:44:55.260 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-09-17 08:42:37.802427807 +0000 2024-09-17T08:44:55.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T08:44:55.326 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-09-17T08:44:55.327 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-09-17T08:44:55.327 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000216892 s, 2.4 MB/s 2024-09-17T08:44:55.328 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T08:44:55.386 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_3 2024-09-17T08:44:55.443 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-09-17 08:44:31.009479164 +0000 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-09-17 08:42:38.115419588 +0000 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-09-17 08:42:38.115419588 +0000 2024-09-17T08:44:55.444 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-09-17 08:42:38.115419588 +0000 2024-09-17T08:44:55.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T08:44:55.510 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-09-17T08:44:55.510 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-09-17T08:44:55.510 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000220233 s, 2.3 MB/s 2024-09-17T08:44:55.512 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T08:44:55.570 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_4 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2024-09-17 08:44:31.009479164 +0000 2024-09-17T08:44:55.627 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2024-09-17 08:42:38.451410768 +0000 2024-09-17T08:44:55.628 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2024-09-17 08:42:38.451410768 +0000 2024-09-17T08:44:55.628 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: 2024-09-17 08:42:38.451410768 +0000 2024-09-17T08:44:55.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T08:44:55.693 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2024-09-17T08:44:55.693 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2024-09-17T08:44:55.693 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000199647 s, 2.6 MB/s 2024-09-17T08:44:55.694 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T08:44:55.752 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:44:55.752 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T08:44:55.768 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T08:44:55.768 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-09-17T08:44:55.823 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 08:44:40.353315826 +0000 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 08:42:30.342770815 +0000 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 08:42:30.342770815 +0000 2024-09-17T08:44:55.824 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 08:42:30.342770815 +0000 2024-09-17T08:44:55.824 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T08:44:55.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:54.439154+0000 mgr.y (mgr.14144) 30 : audit [DBG] from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:55.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:54.439974+0000 mgr.y (mgr.14144) 31 : cephadm [INF] Saving service mgr spec with placement smithi027=y;smithi105=x;count:2 2024-09-17T08:44:55.886 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:54.455665+0000 mgr.y (mgr.14144) 32 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:44:55.889 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T08:44:55.890 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T08:44:55.890 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000219566 s, 2.3 MB/s 2024-09-17T08:44:55.891 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T08:44:55.947 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-09-17T08:44:56.002 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T08:44:56.002 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:56.002 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 08:44:40.353315826 +0000 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 08:42:30.671764258 +0000 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 08:42:30.671764258 +0000 2024-09-17T08:44:56.003 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 08:42:30.671764258 +0000 2024-09-17T08:44:56.003 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T08:44:56.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:54.439154+0000 mgr.y (mgr.14144) 30 : audit [DBG] from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:56.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:54.439974+0000 mgr.y (mgr.14144) 31 : cephadm [INF] Saving service mgr spec with placement smithi027=y;smithi105=x;count:2 2024-09-17T08:44:56.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:54.455665+0000 mgr.y (mgr.14144) 32 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:44:56.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:54.439154+0000 mgr.y (mgr.14144) 30 : audit [DBG] from='client.34103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi027=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:44:56.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:54.439974+0000 mgr.y (mgr.14144) 31 : cephadm [INF] Saving service mgr spec with placement smithi027=y;smithi105=x;count:2 2024-09-17T08:44:56.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:54.455665+0000 mgr.y (mgr.14144) 32 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:44:56.066 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T08:44:56.066 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T08:44:56.066 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000213765 s, 2.4 MB/s 2024-09-17T08:44:56.068 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T08:44:56.131 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-09-17T08:44:56.186 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T08:44:56.186 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:56.186 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-17T08:44:56.186 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:56.187 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:56.187 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 08:44:40.353315826 +0000 2024-09-17T08:44:56.187 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 08:42:30.988757941 +0000 2024-09-17T08:44:56.187 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 08:42:30.988757941 +0000 2024-09-17T08:44:56.187 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 08:42:30.988757941 +0000 2024-09-17T08:44:56.187 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T08:44:56.252 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T08:44:56.252 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T08:44:56.252 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000194284 s, 2.6 MB/s 2024-09-17T08:44:56.253 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T08:44:56.309 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-09-17T08:44:56.365 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T08:44:56.365 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T08:44:56.365 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-17T08:44:56.365 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T08:44:56.365 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T08:44:56.366 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-17 08:44:56.359996760 +0000 2024-09-17T08:44:56.366 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-17 08:42:31.302751683 +0000 2024-09-17T08:44:56.366 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-17 08:42:31.302751683 +0000 2024-09-17T08:44:56.366 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-17 08:42:31.302751683 +0000 2024-09-17T08:44:56.366 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T08:44:56.430 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-17T08:44:56.430 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-17T08:44:56.430 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000197267 s, 2.6 MB/s 2024-09-17T08:44:56.432 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T08:44:56.489 INFO:tasks.cephadm:Deploying osd.0 on smithi027 with /dev/vg_nvme/lv_4... 2024-09-17T08:44:56.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-17T08:44:56.798 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:54.760884+0000 mgr.y (mgr.14144) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:56.798 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:54.760884+0000 mgr.y (mgr.14144) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:56.827 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:54.760884+0000 mgr.y (mgr.14144) 33 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:57.277 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:44:56 smithi105 systemd[1]: Started Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:44:58.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:56.761174+0000 mgr.y (mgr.14144) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:58.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:56.947283+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:56.948209+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:58.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:57.840860+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:57.844438+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:57.845746+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:58.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:57.846571+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:58.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:57.847255+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:56.761174+0000 mgr.y (mgr.14144) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:56.947283+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:56.948209+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:57.840860+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:57.844438+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:57.845746+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:58.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:57.846571+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:58.145 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:57.847255+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:56.761174+0000 mgr.y (mgr.14144) 34 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:56.947283+0000 mon.a (mon.0) 175 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:56.948209+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:57.840860+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:57.844438+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:57.845746+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:57.846571+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:44:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:57.847255+0000 mon.a (mon.0) 181 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:44:59.220 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:57.845473+0000 mgr.y (mgr.14144) 35 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-17T08:44:59.220 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:44:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:44:57.847838+0000 mgr.y (mgr.14144) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi027 2024-09-17T08:44:59.221 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:57.845473+0000 mgr.y (mgr.14144) 35 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-17T08:44:59.221 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:44:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:44:57.847838+0000 mgr.y (mgr.14144) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi027 2024-09-17T08:44:59.221 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-17T08:44:59.221 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-17T08:44:59.221 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-09-17T08:44:59.221 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-09-17T08:44:59.221 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112507 s, 932 MB/s 2024-09-17T08:44:59.222 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-09-17T08:44:59.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:57.845473+0000 mgr.y (mgr.14144) 35 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-17T08:44:59.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:44:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:44:57.847838+0000 mgr.y (mgr.14144) 36 : cephadm [INF] Reconfiguring daemon mgr.y on smithi027 2024-09-17T08:44:59.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_4 2024-09-17T08:45:00.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:44:58.761389+0000 mgr.y (mgr.14144) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:00.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:59.290622+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:00.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:59.291294+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:00.497 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:44:59.295312+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:00.497 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:44:58.761389+0000 mgr.y (mgr.14144) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:00.498 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:59.290622+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:00.498 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:59.291294+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:00.498 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:44:59.295312+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:00.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:44:58.761389+0000 mgr.y (mgr.14144) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:00.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:59.290622+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:59.291294+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:44:59.295312+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:02.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:00.761620+0000 mgr.y (mgr.14144) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:02.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:01.729925+0000 mon.a (mon.0) 185 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:45:02.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.730519+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:45:02.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.730877+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:45:02.423 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.731625+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.731841+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.844676+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.846233+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.846803+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:00.761620+0000 mgr.y (mgr.14144) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:01.729925+0000 mon.a (mon.0) 185 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.730519+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.730877+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.731625+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:45:02.424 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.731841+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:45:02.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.844676+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:02.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.846233+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:02.425 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.846803+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:02.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:00.761620+0000 mgr.y (mgr.14144) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:01.729925+0000 mon.a (mon.0) 185 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:45:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.730519+0000 mon.b (mon.2) 2 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:45:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.730877+0000 mon.b (mon.2) 3 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:45:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.731625+0000 mon.b (mon.2) 4 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:45:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.731841+0000 mon.b (mon.2) 5 : audit [DBG] from='mgr.? 172.21.15.105:0/2079049475' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:45:02.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.844676+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:02.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.846233+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:02.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.846803+0000 mon.a (mon.0) 188 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:03.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:01.843911+0000 mgr.y (mgr.14144) 39 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:03.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:01.846018+0000 mgr.y (mgr.14144) 40 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:02.304212+0000 mon.a (mon.0) 189 : cluster [DBG] mgrmap e14: y(active, since 51s), standbys: x 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:02.304320+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:01.843911+0000 mgr.y (mgr.14144) 39 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:01.846018+0000 mgr.y (mgr.14144) 40 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:02.304212+0000 mon.a (mon.0) 189 : cluster [DBG] mgrmap e14: y(active, since 51s), standbys: x 2024-09-17T08:45:03.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:02.304320+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:45:03.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:01.843911+0000 mgr.y (mgr.14144) 39 : audit [DBG] from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:03.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:01.846018+0000 mgr.y (mgr.14144) 40 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:03.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:02.304212+0000 mon.a (mon.0) 189 : cluster [DBG] mgrmap e14: y(active, since 51s), standbys: x 2024-09-17T08:45:03.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:02.304320+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:45:05.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:02.761901+0000 mgr.y (mgr.14144) 41 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:05.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:03.980363+0000 mon.a (mon.0) 191 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]: dispatch 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:03.986473+0000 mon.a (mon.0) 192 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]': finished 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:03.986538+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:03.986639+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:04.410908+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.27:0/3466034825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:02.761901+0000 mgr.y (mgr.14144) 41 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:03.980363+0000 mon.a (mon.0) 191 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]: dispatch 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:03.986473+0000 mon.a (mon.0) 192 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]': finished 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:03.986538+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-17T08:45:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:03.986639+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:05.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:04.410908+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.27:0/3466034825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:05.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:02.761901+0000 mgr.y (mgr.14144) 41 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:05.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:03.980363+0000 mon.a (mon.0) 191 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]: dispatch 2024-09-17T08:45:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:03.986473+0000 mon.a (mon.0) 192 : audit [INF] from='client.? 172.21.15.27:0/1083598451' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "46e1cb43-f6d6-4bc3-bc29-7067084ed835"}]': finished 2024-09-17T08:45:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:03.986538+0000 mon.a (mon.0) 193 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-17T08:45:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:03.986639+0000 mon.a (mon.0) 194 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:04.410908+0000 mon.c (mon.1) 3 : audit [DBG] from='client.? 172.21.15.27:0/3466034825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:07.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:04.762169+0000 mgr.y (mgr.14144) 42 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:07.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:04.762169+0000 mgr.y (mgr.14144) 42 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:07.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:04.762169+0000 mgr.y (mgr.14144) 42 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:08.817 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:06.762427+0000 mgr.y (mgr.14144) 43 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:08.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:06.762427+0000 mgr.y (mgr.14144) 43 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:09.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:06.762427+0000 mgr.y (mgr.14144) 43 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:10.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:08.762700+0000 mgr.y (mgr.14144) 44 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:10.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:09.745298+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T08:45:10.977 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:09.745930+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:10.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:08.762700+0000 mgr.y (mgr.14144) 44 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:10.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:09.745298+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T08:45:10.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:09.745930+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:11.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:08.762700+0000 mgr.y (mgr.14144) 44 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:09.745298+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T08:45:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:09.745930+0000 mon.a (mon.0) 196 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:11.660 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:09.746446+0000 mgr.y (mgr.14144) 45 : cephadm [INF] Deploying daemon osd.0 on smithi027 2024-09-17T08:45:11.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:10.881047+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:45:11.661 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:10.887698+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:45:11.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:09.746446+0000 mgr.y (mgr.14144) 45 : cephadm [INF] Deploying daemon osd.0 on smithi027 2024-09-17T08:45:11.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:10.881047+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:45:11.661 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:10.887698+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:45:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:09.746446+0000 mgr.y (mgr.14144) 45 : cephadm [INF] Deploying daemon osd.0 on smithi027 2024-09-17T08:45:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:10.881047+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:45:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:10.887698+0000 mon.a (mon.0) 198 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:45:12.650 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:10.763034+0000 mgr.y (mgr.14144) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:12.650 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:10.763034+0000 mgr.y (mgr.14144) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:13.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:10.763034+0000 mgr.y (mgr.14144) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:13.180 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 0 on host 'smithi027' 2024-09-17T08:45:13.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:12.763363+0000 mgr.y (mgr.14144) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:13.840 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:13.169769+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:13.173820+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:13.177974+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:12.763363+0000 mgr.y (mgr.14144) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:13.169769+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:13.173820+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:13.841 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:13.177974+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:13.863 DEBUG:teuthology.orchestra.run.smithi027:osd.0> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T08:45:13.865 INFO:tasks.cephadm:Deploying osd.1 on smithi027 with /dev/vg_nvme/lv_3... 2024-09-17T08:45:13.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-17T08:45:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:12.763363+0000 mgr.y (mgr.14144) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:13.169769+0000 mon.a (mon.0) 199 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:13.173820+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:13.177974+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:16.128 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:14.763530+0000 mgr.y (mgr.14144) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:16.128 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:14.763530+0000 mgr.y (mgr.14144) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:14.763530+0000 mgr.y (mgr.14144) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.013601 s, 771 MB/s 2024-09-17T08:45:16.356 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-09-17T08:45:16.422 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 08:45:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T08:45:16.288+0000 7fb0bc595080 -1 osd.0 0 log_to_monitors {default=true} 2024-09-17T08:45:17.076 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:15.843796+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:17.076 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:16.291585+0000 mon.a (mon.0) 203 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T08:45:17.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:15.843796+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:17.076 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:16.291585+0000 mon.a (mon.0) 203 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T08:45:17.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:15.843796+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:17.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:16.291585+0000 mon.a (mon.0) 203 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T08:45:17.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_3 2024-09-17T08:45:17.956 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T08:45:17.854+0000 7fb0b2b06700 -1 osd.0 0 waiting for initial osdmap 2024-09-17T08:45:17.956 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T08:45:17.858+0000 7fb0ad27e700 -1 osd.0 7 set_numa_affinity unable to identify public interface 'ens1f0' numa node: (0) Success 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:16.763753+0000 mgr.y (mgr.14144) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:16.848193+0000 mon.a (mon.0) 204 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:16.848268+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:16.848493+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:16.848899+0000 mon.a (mon.0) 207 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:17.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:16.763753+0000 mgr.y (mgr.14144) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:17.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:16.848193+0000 mon.a (mon.0) 204 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T08:45:17.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:16.848268+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-17T08:45:17.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:16.848493+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:17.958 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:16.848899+0000 mon.a (mon.0) 207 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:16.763753+0000 mgr.y (mgr.14144) 49 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:16.848193+0000 mon.a (mon.0) 204 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T08:45:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:16.848268+0000 mon.a (mon.0) 205 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-17T08:45:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:16.848493+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:16.848899+0000 mon.a (mon.0) 207 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:19.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:17.850493+0000 mon.a (mon.0) 208 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:19.238 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:17.850611+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:17.851330+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:17.856363+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:17.850493+0000 mon.a (mon.0) 208 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:17.850611+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:17.851330+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.239 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:17.856363+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:17.850493+0000 mon.a (mon.0) 208 : audit [INF] from='osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:17.850611+0000 mon.a (mon.0) 209 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-17T08:45:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:17.851330+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:17.856363+0000 mon.a (mon.0) 211 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:19.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:17.321864+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:17.321864+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:17.322065+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:18.764033+0000 mgr.y (mgr.14144) 50 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:18.857072+0000 mon.a (mon.0) 212 : cluster [INF] osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019] boot 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:18.857121+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-17T08:45:20.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:18.857713+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:17.322065+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:18.764033+0000 mgr.y (mgr.14144) 50 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:18.857072+0000 mon.a (mon.0) 212 : cluster [INF] osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019] boot 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:18.857121+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:18.857713+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:17.321864+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:17.322065+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:18.764033+0000 mgr.y (mgr.14144) 50 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:18.857072+0000 mon.a (mon.0) 212 : cluster [INF] osd.0 [v2:172.21.15.27:6802/1678887019,v1:172.21.15.27:6803/1678887019] boot 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:18.857121+0000 mon.a (mon.0) 213 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-17T08:45:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:18.857713+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:19.863371+0000 mon.a (mon.0) 215 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:19.984636+0000 mgr.y (mgr.14144) 51 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:19.985340+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:19.986332+0000 mgr.y (mgr.14144) 52 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:19.986528+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:19.987055+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:20.587117+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:20.816689+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:20.817777+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:20.821628+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:19.863371+0000 mon.a (mon.0) 215 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:19.984636+0000 mgr.y (mgr.14144) 51 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:19.985340+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:19.986332+0000 mgr.y (mgr.14144) 52 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:19.986528+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:19.987055+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:20.587117+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:20.816689+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:20.817777+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:21.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:20.821628+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:19.863371+0000 mon.a (mon.0) 215 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-17T08:45:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:19.984636+0000 mgr.y (mgr.14144) 51 : audit [DBG] from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:19.985340+0000 mon.a (mon.0) 216 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:19.986332+0000 mgr.y (mgr.14144) 52 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:19.986528+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:21.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:19.987055+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:21.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:20.587117+0000 mon.a (mon.0) 219 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:20.816689+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:21.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:20.817777+0000 mon.a (mon.0) 221 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:21.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:20.821628+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:22.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:20.582933+0000 mgr.y (mgr.14144) 53 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:22.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:20.764308+0000 mgr.y (mgr.14144) 54 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:22.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:20.582933+0000 mgr.y (mgr.14144) 53 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:22.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:20.764308+0000 mgr.y (mgr.14144) 54 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:20.582933+0000 mgr.y (mgr.14144) 53 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:20.764308+0000 mgr.y (mgr.14144) 54 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:22.415932+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]: dispatch 2024-09-17T08:45:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:22.419356+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]': finished 2024-09-17T08:45:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:22.419422+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-17T08:45:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:22.419527+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:22.859993+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.27:0/3538900742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:22.415932+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]: dispatch 2024-09-17T08:45:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:22.419356+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]': finished 2024-09-17T08:45:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:22.419422+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-17T08:45:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:22.419527+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:22.859993+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.27:0/3538900742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:23.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:22.415932+0000 mon.a (mon.0) 223 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]: dispatch 2024-09-17T08:45:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:22.419356+0000 mon.a (mon.0) 224 : audit [INF] from='client.? 172.21.15.27:0/2790642265' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "40dcd331-2a54-4ef7-8d4e-5117cbca6438"}]': finished 2024-09-17T08:45:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:22.419422+0000 mon.a (mon.0) 225 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-17T08:45:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:22.419527+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:22.859993+0000 mon.c (mon.1) 4 : audit [DBG] from='client.? 172.21.15.27:0/3538900742' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:25.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:22.764549+0000 mgr.y (mgr.14144) 55 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:25.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:22.764549+0000 mgr.y (mgr.14144) 55 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:25.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:22.764549+0000 mgr.y (mgr.14144) 55 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:26.740 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:24.764802+0000 mgr.y (mgr.14144) 56 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:26.741 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:24.764802+0000 mgr.y (mgr.14144) 56 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:27.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:24.764802+0000 mgr.y (mgr.14144) 56 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:28.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:26.765099+0000 mgr.y (mgr.14144) 57 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:28.737 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:28.414400+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T08:45:28.738 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:28.415036+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:28.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:26.765099+0000 mgr.y (mgr.14144) 57 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:28.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:28.414400+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T08:45:28.738 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:28.415036+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:29.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:26.765099+0000 mgr.y (mgr.14144) 57 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:28.414400+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T08:45:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:28.415036+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:29.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:28.415551+0000 mgr.y (mgr.14144) 58 : cephadm [INF] Deploying daemon osd.1 on smithi027 2024-09-17T08:45:29.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:28.415551+0000 mgr.y (mgr.14144) 58 : cephadm [INF] Deploying daemon osd.1 on smithi027 2024-09-17T08:45:30.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:28.415551+0000 mgr.y (mgr.14144) 58 : cephadm [INF] Deploying daemon osd.1 on smithi027 2024-09-17T08:45:30.864 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:28.765426+0000 mgr.y (mgr.14144) 59 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:28.765426+0000 mgr.y (mgr.14144) 59 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:28.765426+0000 mgr.y (mgr.14144) 59 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:30.765719+0000 mgr.y (mgr.14144) 60 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.826 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:30.765719+0000 mgr.y (mgr.14144) 60 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:30.765719+0000 mgr.y (mgr.14144) 60 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:31.826 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 1 on host 'smithi027' 2024-09-17T08:45:32.518 DEBUG:teuthology.orchestra.run.smithi027:osd.1> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T08:45:32.521 INFO:tasks.cephadm:Deploying osd.2 on smithi027 with /dev/vg_nvme/lv_2... 2024-09-17T08:45:32.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-17T08:45:32.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:31.815793+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:32.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:31.819538+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:32.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:31.824541+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:32.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:31.815793+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:32.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:31.819538+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:32.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:31.824541+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:33.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:31.815793+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:33.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:31.819538+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:33.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:31.824541+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:33.843 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:32.766016+0000 mgr.y (mgr.14144) 61 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:33.843 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:32.766016+0000 mgr.y (mgr.14144) 61 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:34.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:32.766016+0000 mgr.y (mgr.14144) 61 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:34.999 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 08:45:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T08:45:34.920+0000 7f182d7ce080 -1 osd.1 0 log_to_monitors {default=true} 2024-09-17T08:45:35.301 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-17T08:45:35.301 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-17T08:45:35.301 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-09-17T08:45:35.301 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-09-17T08:45:35.302 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.013119 s, 799 MB/s 2024-09-17T08:45:35.302 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-09-17T08:45:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:34.766300+0000 mgr.y (mgr.14144) 62 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:34.787898+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:34.925363+0000 mon.a (mon.0) 233 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T08:45:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:34.766300+0000 mgr.y (mgr.14144) 62 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:34.787898+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:34.925363+0000 mon.a (mon.0) 233 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T08:45:36.262 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_2 2024-09-17T08:45:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:34.766300+0000 mgr.y (mgr.14144) 62 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:34.787898+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:34.925363+0000 mon.a (mon.0) 233 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T08:45:36.941 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T08:45:36.832+0000 7f1823d3f700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T08:45:36.941 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T08:45:36.836+0000 7f181ccb4700 -1 osd.1 12 set_numa_affinity unable to identify public interface 'ens1f0' numa node: (0) Success 2024-09-17T08:45:36.941 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:35.823689+0000 mon.a (mon.0) 234 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:35.823758+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:35.823869+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:35.824235+0000 mon.a (mon.0) 237 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:35.823689+0000 mon.a (mon.0) 234 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:35.823758+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:35.823869+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:36.942 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:35.824235+0000 mon.a (mon.0) 237 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:37.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:35.823689+0000 mon.a (mon.0) 234 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T08:45:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:35.823758+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-17T08:45:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:35.823869+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:35.824235+0000 mon.a (mon.0) 237 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:38.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:36.766569+0000 mgr.y (mgr.14144) 63 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:38.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:36.830173+0000 mon.a (mon.0) 238 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:38.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:36.830263+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-17T08:45:38.096 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:36.830707+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:36.833865+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:37.833732+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:36.766569+0000 mgr.y (mgr.14144) 63 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:36.830173+0000 mon.a (mon.0) 238 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:36.830263+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:36.830707+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:36.833865+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:37.833732+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:36.766569+0000 mgr.y (mgr.14144) 63 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:36.830173+0000 mon.a (mon.0) 238 : audit [INF] from='osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:36.830263+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-17T08:45:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:36.830707+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:36.833865+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:37.833732+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:35.898872+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:35.898967+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:37.839689+0000 mon.a (mon.0) 243 : cluster [INF] osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426] boot 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:37.839731+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:37.840071+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:38.704488+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:38.705837+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:38.985 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:38.706378+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:35.898872+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:35.898967+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:37.839689+0000 mon.a (mon.0) 243 : cluster [INF] osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426] boot 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:37.839731+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:37.840071+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:38.704488+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:38.705837+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:38.986 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:38.706378+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:35.898872+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:35.898967+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:37.839689+0000 mon.a (mon.0) 243 : cluster [INF] osd.1 [v2:172.21.15.27:6810/2462012426,v1:172.21.15.27:6811/2462012426] boot 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:37.839731+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:37.840071+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:45:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:38.704488+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:45:39.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:38.705837+0000 mon.a (mon.0) 247 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:45:39.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:38.706378+0000 mon.a (mon.0) 248 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:38.703734+0000 mgr.y (mgr.14144) 64 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:38.705643+0000 mgr.y (mgr.14144) 65 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:38.766843+0000 mgr.y (mgr.14144) 66 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:39.849632+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:38.703734+0000 mgr.y (mgr.14144) 64 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:38.705643+0000 mgr.y (mgr.14144) 65 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:38.766843+0000 mgr.y (mgr.14144) 66 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:40.130 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:39.849632+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-17T08:45:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:38.703734+0000 mgr.y (mgr.14144) 64 : audit [DBG] from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:38.705643+0000 mgr.y (mgr.14144) 65 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:45:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:38.766843+0000 mgr.y (mgr.14144) 66 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 4.7 MiB used, 89 GiB / 89 GiB avail 2024-09-17T08:45:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:39.849632+0000 mon.a (mon.0) 249 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:40.340437+0000 mgr.y (mgr.14144) 67 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:40.345069+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:40.348715+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:40.349811+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:40.353732+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:40.340437+0000 mgr.y (mgr.14144) 67 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:40.345069+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:40.348715+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:40.349811+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:41.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:40.353732+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:40.340437+0000 mgr.y (mgr.14144) 67 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:45:41.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:40.345069+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:40.348715+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:41.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:40.349811+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:41.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:40.353732+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:40.767083+0000 mgr.y (mgr.14144) 68 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:41.668267+0000 mon.a (mon.0) 254 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]: dispatch 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:41.672147+0000 mon.a (mon.0) 255 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]': finished 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:41.672210+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:41.672310+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:42.097838+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.27:0/1612940895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:40.767083+0000 mgr.y (mgr.14144) 68 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:41.668267+0000 mon.a (mon.0) 254 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]: dispatch 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:41.672147+0000 mon.a (mon.0) 255 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]': finished 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:41.672210+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:41.672310+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:42.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:42.097838+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.27:0/1612940895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:42.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:40.767083+0000 mgr.y (mgr.14144) 68 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:42.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:41.668267+0000 mon.a (mon.0) 254 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]: dispatch 2024-09-17T08:45:42.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:41.672147+0000 mon.a (mon.0) 255 : audit [INF] from='client.? 172.21.15.27:0/294246066' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "09227eb6-9bec-49b0-a1e8-a954c1872265"}]': finished 2024-09-17T08:45:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:41.672210+0000 mon.a (mon.0) 256 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-17T08:45:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:41.672310+0000 mon.a (mon.0) 257 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:42.097838+0000 mon.c (mon.1) 5 : audit [DBG] from='client.? 172.21.15.27:0/1612940895' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:45:44.664 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:42.767281+0000 mgr.y (mgr.14144) 69 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:44.665 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:42.767281+0000 mgr.y (mgr.14144) 69 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:45.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:42.767281+0000 mgr.y (mgr.14144) 69 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:46.940 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:44.767565+0000 mgr.y (mgr.14144) 70 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:46.941 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:44.767565+0000 mgr.y (mgr.14144) 70 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:47.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:44.767565+0000 mgr.y (mgr.14144) 70 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 9.5 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:47.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:47.502206+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T08:45:47.825 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:47.502892+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:47.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:47.502206+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T08:45:47.826 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:47.502892+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:48.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:47.502206+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T08:45:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:47.502892+0000 mon.a (mon.0) 259 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:45:48.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:46.767840+0000 mgr.y (mgr.14144) 71 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:48.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:47.503422+0000 mgr.y (mgr.14144) 72 : cephadm [INF] Deploying daemon osd.2 on smithi027 2024-09-17T08:45:48.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:46.767840+0000 mgr.y (mgr.14144) 71 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:48.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:47.503422+0000 mgr.y (mgr.14144) 72 : cephadm [INF] Deploying daemon osd.2 on smithi027 2024-09-17T08:45:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:46.767840+0000 mgr.y (mgr.14144) 71 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:47.503422+0000 mgr.y (mgr.14144) 72 : cephadm [INF] Deploying daemon osd.2 on smithi027 2024-09-17T08:45:49.829 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:48.768177+0000 mgr.y (mgr.14144) 73 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:49.829 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:48.768177+0000 mgr.y (mgr.14144) 73 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:50.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:48.768177+0000 mgr.y (mgr.14144) 73 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:51.016 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 2 on host 'smithi027' 2024-09-17T08:45:51.638 DEBUG:teuthology.orchestra.run.smithi027:osd.2> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T08:45:51.645 INFO:tasks.cephadm:Deploying osd.3 on smithi027 with /dev/vg_nvme/lv_1... 2024-09-17T08:45:51.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-17T08:45:52.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:50.768444+0000 mgr.y (mgr.14144) 74 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:52.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:51.002465+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:52.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:51.008041+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:51.014202+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:50.768444+0000 mgr.y (mgr.14144) 74 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:51.002465+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:51.008041+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:51.014202+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:52.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:50.768444+0000 mgr.y (mgr.14144) 74 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:51.002465+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:51.008041+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:51.014202+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:54.274 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-17T08:45:54.275 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-17T08:45:54.275 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2024-09-17T08:45:54.275 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2024-09-17T08:45:54.275 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126358 s, 830 MB/s 2024-09-17T08:45:54.275 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2024-09-17T08:45:54.371 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T08:45:54.113+0000 7f41031c5080 -1 osd.2 0 log_to_monitors {default=true} 2024-09-17T08:45:54.756 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:52.768777+0000 mgr.y (mgr.14144) 75 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:54.756 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:54.080708+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:54.116734+0000 mon.c (mon.1) 6 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:54.117057+0000 mon.a (mon.0) 264 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:52.768777+0000 mgr.y (mgr.14144) 75 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:54.080708+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:54.116734+0000 mon.c (mon.1) 6 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:54.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:54.117057+0000 mon.a (mon.0) 264 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:52.768777+0000 mgr.y (mgr.14144) 75 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:54.080708+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:54.116734+0000 mon.c (mon.1) 6 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:54.117057+0000 mon.a (mon.0) 264 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T08:45:56.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi027:vg_nvme/lv_1 2024-09-17T08:45:56.352 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T08:45:56.090+0000 7f40f9736700 -1 osd.2 0 waiting for initial osdmap 2024-09-17T08:45:56.352 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T08:45:56.094+0000 7f40f5eb2700 -1 osd.2 17 set_numa_affinity unable to identify public interface 'ens1f0' numa node: (0) Success 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:54.769007+0000 mgr.y (mgr.14144) 76 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:55.084291+0000 mon.a (mon.0) 265 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:55.084375+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:55.084504+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:55.084964+0000 mon.c (mon.1) 7 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:55.085237+0000 mon.a (mon.0) 268 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:54.769007+0000 mgr.y (mgr.14144) 76 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:56.353 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:55.084291+0000 mon.a (mon.0) 265 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T08:45:56.354 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:55.084375+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-17T08:45:56.354 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:55.084504+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:56.354 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:55.084964+0000 mon.c (mon.1) 7 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:56.354 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:55.085237+0000 mon.a (mon.0) 268 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:54.769007+0000 mgr.y (mgr.14144) 76 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:55.084291+0000 mon.a (mon.0) 265 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:55.084375+0000 mon.a (mon.0) 266 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:55.084504+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:55.084964+0000 mon.c (mon.1) 7 : audit [INF] from='osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:56.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:55.085237+0000 mon.a (mon.0) 268 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:45:57.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:56.085151+0000 mon.a (mon.0) 269 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:57.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:56.085263+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-17T08:45:57.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:56.085665+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:57.356 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:56.091378+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:57.357 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:56.085151+0000 mon.a (mon.0) 269 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:57.357 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:56.085263+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-17T08:45:57.357 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:56.085665+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:57.357 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:56.091378+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:57.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:56.085151+0000 mon.a (mon.0) 269 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:45:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:56.085263+0000 mon.a (mon.0) 270 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-17T08:45:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:56.085665+0000 mon.a (mon.0) 271 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:56.091378+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:55.114263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:55.114363+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:56.769313+0000 mgr.y (mgr.14144) 77 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:58.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:57.088895+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:57.094821+0000 mon.a (mon.0) 274 : cluster [INF] osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382] boot 2024-09-17T08:45:58.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:57.094866+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-17T08:45:58.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:57.094970+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:57.095355+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-09-17T08:45:58.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:55.114263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:58.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:55.114363+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:58.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:56.769313+0000 mgr.y (mgr.14144) 77 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:58.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:57.088895+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:57.094821+0000 mon.a (mon.0) 274 : cluster [INF] osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382] boot 2024-09-17T08:45:58.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:57.094866+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-17T08:45:58.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:57.094970+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:57.095355+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-09-17T08:45:58.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:55.114263+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:55.114363+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:56.769313+0000 mgr.y (mgr.14144) 77 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 9.6 MiB used, 179 GiB / 179 GiB avail 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:57.088895+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:57.094821+0000 mon.a (mon.0) 274 : cluster [INF] osd.2 [v2:172.21.15.27:6818/3166119382,v1:172.21.15.27:6819/3166119382] boot 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:57.094866+0000 mon.a (mon.0) 275 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:57.094970+0000 mon.a (mon.0) 276 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:57.095355+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]: dispatch 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.099920+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:58.099975+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.100792+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.195690+0000 mgr.y (mgr.14144) 78 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.613004+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.616699+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.617863+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:59.385 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:58.621320+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.099920+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:58.099975+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.100792+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.195690+0000 mgr.y (mgr.14144) 78 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.613004+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.616699+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.617863+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:59.386 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:45:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:58.621320+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.099920+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1}]': finished 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:58.099975+0000 mon.a (mon.0) 279 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.100792+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.195690+0000 mgr.y (mgr.14144) 78 : audit [DBG] from='client.14283 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.613004+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.616699+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.617863+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:45:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:45:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:58.621320+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:00.377 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:58.608096+0000 mgr.y (mgr.14144) 79 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:58.769756+0000 mgr.y (mgr.14144) 80 : cluster [DBG] pgmap v63: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:59.101739+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:45:59.101855+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:59.108001+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:45:59.109256+0000 mgr.y (mgr.14144) 81 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:59.109493+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:45:59.110161+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:58.608096+0000 mgr.y (mgr.14144) 79 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:58.769756+0000 mgr.y (mgr.14144) 80 : cluster [DBG] pgmap v63: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:00.378 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:59.101739+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-09-17T08:46:00.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:45:59.101855+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-17T08:46:00.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:59.108001+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:00.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:45:59.109256+0000 mgr.y (mgr.14144) 81 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:46:00.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:59.109493+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:00.379 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:45:59.110161+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:00.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:58.608096+0000 mgr.y (mgr.14144) 79 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:58.769756+0000 mgr.y (mgr.14144) 80 : cluster [DBG] pgmap v63: 1 pgs: 1 unknown; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:59.101739+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:45:59.101855+0000 mon.a (mon.0) 286 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:59.108001+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:45:59.109256+0000 mgr.y (mgr.14144) 81 : cephadm [INF] Applying service osd.None on host smithi027... 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:59.109493+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:45:59.110161+0000 mon.a (mon.0) 289 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:00.105341+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-17T08:46:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:00.105341+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-17T08:46:01.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:00.105341+0000 mon.a (mon.0) 290 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-17T08:46:02.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:00.770077+0000 mgr.y (mgr.14144) 82 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:02.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:01.387099+0000 mon.c (mon.1) 8 : audit [INF] from='client.? 172.21.15.27:0/2436374292' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:01.387406+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:01.392712+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]': finished 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:01.392776+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:01.392870+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:01.816630+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.27:0/1870613223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:00.770077+0000 mgr.y (mgr.14144) 82 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:01.387099+0000 mon.c (mon.1) 8 : audit [INF] from='client.? 172.21.15.27:0/2436374292' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:01.387406+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:01.392712+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]': finished 2024-09-17T08:46:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:01.392776+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-17T08:46:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:01.392870+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:01.816630+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.27:0/1870613223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:02.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:00.770077+0000 mgr.y (mgr.14144) 82 : cluster [DBG] pgmap v66: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:02.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:01.387099+0000 mon.c (mon.1) 8 : audit [INF] from='client.? 172.21.15.27:0/2436374292' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:01.387406+0000 mon.a (mon.0) 291 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]: dispatch 2024-09-17T08:46:02.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:01.392712+0000 mon.a (mon.0) 292 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e66dcd4-23e5-42b7-b665-8fa128130229"}]': finished 2024-09-17T08:46:02.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:01.392776+0000 mon.a (mon.0) 293 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-17T08:46:02.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:01.392870+0000 mon.a (mon.0) 294 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:02.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:01.816630+0000 mon.a (mon.0) 295 : audit [DBG] from='client.? 172.21.15.27:0/1870613223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:04.752 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:02.770424+0000 mgr.y (mgr.14144) 83 : cluster [DBG] pgmap v68: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:04.752 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:02.770424+0000 mgr.y (mgr.14144) 83 : cluster [DBG] pgmap v68: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:05.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:02.770424+0000 mgr.y (mgr.14144) 83 : cluster [DBG] pgmap v68: 1 pgs: 1 creating+peering; 0 B data, 14 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:06.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:04.770756+0000 mgr.y (mgr.14144) 84 : cluster [DBG] pgmap v69: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:04.770756+0000 mgr.y (mgr.14144) 84 : cluster [DBG] pgmap v69: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:06.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:04.770756+0000 mgr.y (mgr.14144) 84 : cluster [DBG] pgmap v69: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:06.770995+0000 mgr.y (mgr.14144) 85 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:07.424828+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T08:46:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:07.425516+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:07.426075+0000 mgr.y (mgr.14144) 86 : cephadm [INF] Deploying daemon osd.3 on smithi027 2024-09-17T08:46:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:06.770995+0000 mgr.y (mgr.14144) 85 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:07.424828+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T08:46:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:07.425516+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:07.426075+0000 mgr.y (mgr.14144) 86 : cephadm [INF] Deploying daemon osd.3 on smithi027 2024-09-17T08:46:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:06.770995+0000 mgr.y (mgr.14144) 85 : cluster [DBG] pgmap v70: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:07.424828+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T08:46:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:07.425516+0000 mon.a (mon.0) 297 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:07.426075+0000 mgr.y (mgr.14144) 86 : cephadm [INF] Deploying daemon osd.3 on smithi027 2024-09-17T08:46:10.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:08.771253+0000 mgr.y (mgr.14144) 87 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:10.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:08.771253+0000 mgr.y (mgr.14144) 87 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:08.771253+0000 mgr.y (mgr.14144) 87 : cluster [DBG] pgmap v71: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:10.979 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 3 on host 'smithi027' 2024-09-17T08:46:11.626 DEBUG:teuthology.orchestra.run.smithi027:osd.3> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T08:46:11.628 INFO:tasks.cephadm:Deploying osd.4 on smithi105 with /dev/vg_nvme/lv_4... 2024-09-17T08:46:11.628 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-17T08:46:11.919 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:10.771484+0000 mgr.y (mgr.14144) 88 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:10.881923+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:10.888447+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:10.964868+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:10.970404+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:10.976915+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:10.771484+0000 mgr.y (mgr.14144) 88 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:11.920 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:10.881923+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:46:11.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:10.888447+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:46:11.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:10.964868+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:11.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:10.970404+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:11.921 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:10.976915+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:10.771484+0000 mgr.y (mgr.14144) 88 : cluster [DBG] pgmap v72: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:10.881923+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:10.888447+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:10.964868+0000 mon.a (mon.0) 300 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:10.970404+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:11.929 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:10.976915+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:13.535 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0116142 s, 903 MB/s 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 2024-09-17T08:46:13.536 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-09-17T08:46:13.712 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:12.525053+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:13.713 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:12.525053+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:13.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:12.525053+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:14.052 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_4 2024-09-17T08:46:14.257 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 08:46:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T08:46:14.136+0000 7f548df80080 -1 osd.3 0 log_to_monitors {default=true} 2024-09-17T08:46:14.860 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:12.771721+0000 mgr.y (mgr.14144) 89 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:14.861 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:14.141092+0000 mon.a (mon.0) 304 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T08:46:14.861 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:12.771721+0000 mgr.y (mgr.14144) 89 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:14.861 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:14.141092+0000 mon.a (mon.0) 304 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T08:46:14.938 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:12.771721+0000 mgr.y (mgr.14144) 89 : cluster [DBG] pgmap v73: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:14.938 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:14.141092+0000 mon.a (mon.0) 304 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T08:46:16.009 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T08:46:15.574+0000 7f54844f1700 -1 osd.3 0 waiting for initial osdmap 2024-09-17T08:46:16.009 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T08:46:15.581+0000 7f547d466700 -1 osd.3 24 set_numa_affinity unable to identify public interface 'ens1f0' numa node: (0) Success 2024-09-17T08:46:16.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:14.569784+0000 mon.a (mon.0) 305 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T08:46:16.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:14.569858+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-17T08:46:16.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:14.570252+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:14.570580+0000 mon.a (mon.0) 308 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:14.771985+0000 mgr.y (mgr.14144) 90 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:14.569784+0000 mon.a (mon.0) 305 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:14.569858+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:14.570252+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:14.570580+0000 mon.a (mon.0) 308 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:46:16.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:14.771985+0000 mgr.y (mgr.14144) 90 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:16.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:14.569784+0000 mon.a (mon.0) 305 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T08:46:16.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:14.569858+0000 mon.a (mon.0) 306 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-17T08:46:16.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:14.570252+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:16.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:14.570580+0000 mon.a (mon.0) 308 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T08:46:16.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:14.771985+0000 mgr.y (mgr.14144) 90 : cluster [DBG] pgmap v75: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:16.985 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.572357+0000 mon.a (mon.0) 309 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:46:16.985 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:15.572465+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-17T08:46:16.985 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.572579+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:16.985 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.578973+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:15.629429+0000 mgr.y (mgr.14144) 91 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.633628+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.639318+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.640509+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.647229+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.742636+0000 mgr.y (mgr.14144) 92 : audit [DBG] from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.743201+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:15.744295+0000 mgr.y (mgr.14144) 93 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.744494+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:15.745049+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:16.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:16.575836+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.572357+0000 mon.a (mon.0) 309 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:46:17.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:15.572465+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.572579+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.578973+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:15.629429+0000 mgr.y (mgr.14144) 91 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.633628+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.639318+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.640509+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.647229+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.742636+0000 mgr.y (mgr.14144) 92 : audit [DBG] from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.743201+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:15.744295+0000 mgr.y (mgr.14144) 93 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.744494+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:15.745049+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:16.575836+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.572357+0000 mon.a (mon.0) 309 : audit [INF] from='osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:15.572465+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.572579+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.578973+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:15.629429+0000 mgr.y (mgr.14144) 91 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.633628+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.639318+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.640509+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.647229+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.742636+0000 mgr.y (mgr.14144) 92 : audit [DBG] from='client.34193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.743201+0000 mon.a (mon.0) 317 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:15.744295+0000 mgr.y (mgr.14144) 93 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.744494+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:15.745049+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:16.575836+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:18.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:15.159994+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:15.160083+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:16.590179+0000 mon.a (mon.0) 321 : cluster [INF] osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975] boot 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:16.590213+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:16.590838+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:16.772306+0000 mgr.y (mgr.14144) 94 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:15.159994+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:15.160083+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:16.590179+0000 mon.a (mon.0) 321 : cluster [INF] osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975] boot 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:16.590213+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-17T08:46:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:16.590838+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:18.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:16.772306+0000 mgr.y (mgr.14144) 94 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:15.159994+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:15.160083+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:16.590179+0000 mon.a (mon.0) 321 : cluster [INF] osd.3 [v2:172.21.15.27:6826/1975358975,v1:172.21.15.27:6827/1975358975] boot 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:16.590213+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:16.590838+0000 mon.a (mon.0) 323 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:46:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:16.772306+0000 mgr.y (mgr.14144) 94 : cluster [DBG] pgmap v78: 1 pgs: 1 active+clean; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T08:46:19.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:17.607534+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:17.953335+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.105:0/158915407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:17.953988+0000 mon.a (mon.0) 325 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:17.959011+0000 mon.a (mon.0) 326 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]': finished 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:17.959113+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:17.959333+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:18.358645+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.105:0/2116900533' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:17.607534+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-17T08:46:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:17.953335+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.105:0/158915407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:17.953988+0000 mon.a (mon.0) 325 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:17.959011+0000 mon.a (mon.0) 326 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]': finished 2024-09-17T08:46:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:17.959113+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-17T08:46:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:17.959333+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:18.358645+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.105:0/2116900533' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:19.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:17.607534+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:17.953335+0000 mon.b (mon.2) 6 : audit [INF] from='client.? 172.21.15.105:0/158915407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:17.953988+0000 mon.a (mon.0) 325 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]: dispatch 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:17.959011+0000 mon.a (mon.0) 326 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "34b2e281-2987-41a2-adc6-fb895ded7659"}]': finished 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:17.959113+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:17.959333+0000 mon.a (mon.0) 328 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:18.358645+0000 mon.b (mon.2) 7 : audit [DBG] from='client.? 172.21.15.105:0/2116900533' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:20.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:18.772564+0000 mgr.y (mgr.14144) 95 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:20.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:18.772564+0000 mgr.y (mgr.14144) 95 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:20.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:18.772564+0000 mgr.y (mgr.14144) 95 : cluster [DBG] pgmap v81: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:22.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:20.773076+0000 mgr.y (mgr.14144) 96 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:22.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:20.773076+0000 mgr.y (mgr.14144) 96 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:22.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:20.773076+0000 mgr.y (mgr.14144) 96 : cluster [DBG] pgmap v82: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:23.518811+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T08:46:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:23.519649+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:24.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:23.518811+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T08:46:24.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:23.519649+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:23.518811+0000 mon.a (mon.0) 329 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T08:46:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:23.519649+0000 mon.a (mon.0) 330 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:24.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:22.773361+0000 mgr.y (mgr.14144) 97 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:24.696 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:23.520328+0000 mgr.y (mgr.14144) 98 : cephadm [INF] Deploying daemon osd.4 on smithi105 2024-09-17T08:46:25.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:22.773361+0000 mgr.y (mgr.14144) 97 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:25.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:23.520328+0000 mgr.y (mgr.14144) 98 : cephadm [INF] Deploying daemon osd.4 on smithi105 2024-09-17T08:46:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:22.773361+0000 mgr.y (mgr.14144) 97 : cluster [DBG] pgmap v83: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:23.520328+0000 mgr.y (mgr.14144) 98 : cephadm [INF] Deploying daemon osd.4 on smithi105 2024-09-17T08:46:26.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:24.773756+0000 mgr.y (mgr.14144) 99 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:26.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:24.773756+0000 mgr.y (mgr.14144) 99 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:24.773756+0000 mgr.y (mgr.14144) 99 : cluster [DBG] pgmap v84: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:26.787 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 4 on host 'smithi105' 2024-09-17T08:46:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:26.774563+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:26.779223+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:26.785527+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:26.774563+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:26.779223+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:26.785527+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:26.774563+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:26.779223+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:26.785527+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:27.377 DEBUG:teuthology.orchestra.run.smithi105:osd.4> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T08:46:27.380 INFO:tasks.cephadm:Deploying osd.5 on smithi105 with /dev/vg_nvme/lv_3... 2024-09-17T08:46:27.380 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-17T08:46:28.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:26.774029+0000 mgr.y (mgr.14144) 100 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:29.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:26.774029+0000 mgr.y (mgr.14144) 100 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:29.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:26.774029+0000 mgr.y (mgr.14144) 100 : cluster [DBG] pgmap v85: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124775 s, 840 MB/s 2024-09-17T08:46:29.510 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-09-17T08:46:30.112 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 08:46:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T08:46:29.875+0000 7fa339d01080 -1 osd.4 0 log_to_monitors {default=true} 2024-09-17T08:46:30.134 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_3 2024-09-17T08:46:30.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:28.774506+0000 mgr.y (mgr.14144) 101 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:30.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:29.879338+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:30.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:29.879686+0000 mon.a (mon.0) 334 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:30.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:30.279844+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:31.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:28.774506+0000 mgr.y (mgr.14144) 101 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:29.879338+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:29.879686+0000 mon.a (mon.0) 334 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:30.279844+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:28.774506+0000 mgr.y (mgr.14144) 101 : cluster [DBG] pgmap v86: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:29.879338+0000 mon.b (mon.2) 8 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:29.879686+0000 mon.a (mon.0) 334 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T08:46:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:30.279844+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:31.922 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T08:46:31.582+0000 7fa330272700 -1 osd.4 0 waiting for initial osdmap 2024-09-17T08:46:31.922 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T08:46:31.588+0000 7fa32a1e9700 -1 osd.4 29 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:30.574326+0000 mon.a (mon.0) 336 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:30.574447+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:30.574869+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:30.575063+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:30.575449+0000 mon.a (mon.0) 339 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:31.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:30.774796+0000 mgr.y (mgr.14144) 102 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:32.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:30.574326+0000 mon.a (mon.0) 336 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:30.574447+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:30.574869+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:30.575063+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:30.575449+0000 mon.a (mon.0) 339 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:30.774796+0000 mgr.y (mgr.14144) 102 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:30.574326+0000 mon.a (mon.0) 336 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:30.574447+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:30.574869+0000 mon.a (mon.0) 338 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:30.575063+0000 mon.b (mon.2) 9 : audit [INF] from='osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:32.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:30.575449+0000 mon.a (mon.0) 339 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:32.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:30.774796+0000 mgr.y (mgr.14144) 102 : cluster [DBG] pgmap v88: 1 pgs: 1 active+clean; 0 B data, 20 MiB used, 358 GiB / 358 GiB avail 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:31.576220+0000 mon.a (mon.0) 340 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:31.576345+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:31.576557+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:31.581613+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:32.570445+0000 mon.a (mon.0) 344 : cluster [INF] osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191] boot 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:32.570503+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-17T08:46:32.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:32.570995+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:31.576220+0000 mon.a (mon.0) 340 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:31.576345+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:31.576557+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:31.581613+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:32.570445+0000 mon.a (mon.0) 344 : cluster [INF] osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191] boot 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:32.570503+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:32.570995+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:31.576220+0000 mon.a (mon.0) 340 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:31.576345+0000 mon.a (mon.0) 341 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:31.576557+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:31.581613+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:32.570445+0000 mon.a (mon.0) 344 : cluster [INF] osd.4 [v2:172.21.15.105:6800/4256538191,v1:172.21.15.105:6801/4256538191] boot 2024-09-17T08:46:33.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:32.570503+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-17T08:46:33.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:32.570995+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:46:33.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:30.830207+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:33.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:30.830352+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:33.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:32.775045+0000 mgr.y (mgr.14144) 103 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:33.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:33.572095+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-17T08:46:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:30.830207+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:30.830352+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:32.775045+0000 mgr.y (mgr.14144) 103 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:33.572095+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-17T08:46:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:30.830207+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:30.830352+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:32.775045+0000 mgr.y (mgr.14144) 103 : cluster [DBG] pgmap v91: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:33.572095+0000 mon.a (mon.0) 347 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-17T08:46:34.909 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:33.621835+0000 mgr.y (mgr.14144) 104 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:34.909 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:33.622746+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:34.909 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:33.624208+0000 mgr.y (mgr.14144) 105 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:33.624469+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:33.625233+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:33.966272+0000 mgr.y (mgr.14144) 106 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:33.971029+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:34.191594+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:34.193095+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:34.197993+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:34.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:34.574705+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:33.621835+0000 mgr.y (mgr.14144) 104 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:33.622746+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:33.624208+0000 mgr.y (mgr.14144) 105 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:33.624469+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:33.625233+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:33.966272+0000 mgr.y (mgr.14144) 106 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:33.971029+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:34.191594+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:34.193095+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:34.197993+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:34.574705+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:33.621835+0000 mgr.y (mgr.14144) 104 : audit [DBG] from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:33.622746+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:33.624208+0000 mgr.y (mgr.14144) 105 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:33.624469+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:33.625233+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:33.966272+0000 mgr.y (mgr.14144) 106 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:33.971029+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:34.191594+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:34.193095+0000 mon.a (mon.0) 353 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:35.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:34.197993+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:35.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:34.574705+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-17T08:46:36.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:34.775330+0000 mgr.y (mgr.14144) 107 : cluster [DBG] pgmap v94: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:34.775330+0000 mgr.y (mgr.14144) 107 : cluster [DBG] pgmap v94: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:36.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:34.775330+0000 mgr.y (mgr.14144) 107 : cluster [DBG] pgmap v94: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:37.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:35.735839+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.105:0/4073692561' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:35.736472+0000 mon.a (mon.0) 356 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:35.740680+0000 mon.a (mon.0) 357 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]': finished 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:35.740759+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:35.740890+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:36.148167+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.105:0/893143519' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:35.735839+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.105:0/4073692561' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:35.736472+0000 mon.a (mon.0) 356 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:35.740680+0000 mon.a (mon.0) 357 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]': finished 2024-09-17T08:46:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:35.740759+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-17T08:46:37.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:35.740890+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:37.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:36.148167+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.105:0/893143519' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:35.735839+0000 mon.b (mon.2) 10 : audit [INF] from='client.? 172.21.15.105:0/4073692561' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:35.736472+0000 mon.a (mon.0) 356 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]: dispatch 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:35.740680+0000 mon.a (mon.0) 357 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73eb3b08-0c61-45b2-a7c2-ceac17ef7982"}]': finished 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:35.740759+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:35.740890+0000 mon.a (mon.0) 359 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:36.148167+0000 mon.b (mon.2) 11 : audit [DBG] from='client.? 172.21.15.105:0/893143519' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:38.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:36.775614+0000 mgr.y (mgr.14144) 108 : cluster [DBG] pgmap v96: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:39.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:36.775614+0000 mgr.y (mgr.14144) 108 : cluster [DBG] pgmap v96: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:36.775614+0000 mgr.y (mgr.14144) 108 : cluster [DBG] pgmap v96: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:39.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:38.775923+0000 mgr.y (mgr.14144) 109 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:40.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:38.775923+0000 mgr.y (mgr.14144) 109 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:38.775923+0000 mgr.y (mgr.14144) 109 : cluster [DBG] pgmap v97: 1 pgs: 1 peering; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:42.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:40.776306+0000 mgr.y (mgr.14144) 110 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:41.250010+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:41.250839+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:41.251520+0000 mgr.y (mgr.14144) 111 : cephadm [INF] Deploying daemon osd.5 on smithi105 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:40.776306+0000 mgr.y (mgr.14144) 110 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:41.250010+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:41.250839+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:41.251520+0000 mgr.y (mgr.14144) 111 : cephadm [INF] Deploying daemon osd.5 on smithi105 2024-09-17T08:46:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:40.776306+0000 mgr.y (mgr.14144) 110 : cluster [DBG] pgmap v98: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:41.250010+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T08:46:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:41.250839+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:41.251520+0000 mgr.y (mgr.14144) 111 : cephadm [INF] Deploying daemon osd.5 on smithi105 2024-09-17T08:46:44.514 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 5 on host 'smithi105' 2024-09-17T08:46:44.673 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:42.776578+0000 mgr.y (mgr.14144) 112 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:44.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:44.501374+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:44.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:44.506462+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:44.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:44.512030+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:42.776578+0000 mgr.y (mgr.14144) 112 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:44.501374+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:44.506462+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:44.512030+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:42.776578+0000 mgr.y (mgr.14144) 112 : cluster [DBG] pgmap v99: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:44.501374+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:44.506462+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:44.512030+0000 mon.a (mon.0) 364 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:45.162 DEBUG:teuthology.orchestra.run.smithi105:osd.5> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T08:46:45.164 INFO:tasks.cephadm:Deploying osd.6 on smithi105 with /dev/vg_nvme/lv_2... 2024-09-17T08:46:45.164 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-17T08:46:45.707 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:44.776876+0000 mgr.y (mgr.14144) 113 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:46.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:44.776876+0000 mgr.y (mgr.14144) 113 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:44.776876+0000 mgr.y (mgr.14144) 113 : cluster [DBG] pgmap v100: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:47.644 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-17T08:46:47.645 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-17T08:46:47.645 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-09-17T08:46:47.645 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-09-17T08:46:47.645 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0127691 s, 821 MB/s 2024-09-17T08:46:47.645 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-09-17T08:46:48.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 08:46:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T08:46:47.624+0000 7f5673c30080 -1 osd.5 0 log_to_monitors {default=true} 2024-09-17T08:46:48.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:46.777314+0000 mgr.y (mgr.14144) 114 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:47.431437+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:47.627851+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:47.628217+0000 mon.a (mon.0) 366 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:46.777314+0000 mgr.y (mgr.14144) 114 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:47.431437+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:47.627851+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:47.628217+0000 mon.a (mon.0) 366 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.764 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:46.777314+0000 mgr.y (mgr.14144) 114 : cluster [DBG] pgmap v101: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:48.764 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:47.431437+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:48.764 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:47.627851+0000 mon.b (mon.2) 12 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.765 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:47.628217+0000 mon.a (mon.0) 366 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T08:46:48.788 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_2 2024-09-17T08:46:49.778 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T08:46:49.587+0000 7f566a1a1700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T08:46:49.778 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T08:46:49.591+0000 7f566591b700 -1 osd.5 35 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-09-17T08:46:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:48.580713+0000 mon.a (mon.0) 367 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T08:46:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:48.580780+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-17T08:46:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:48.580957+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:49.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:48.581293+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:49.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:48.581695+0000 mon.a (mon.0) 370 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:49.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:48.777764+0000 mgr.y (mgr.14144) 115 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:50.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:48.580713+0000 mon.a (mon.0) 367 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:48.580780+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:48.580957+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:48.581293+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:48.581695+0000 mon.a (mon.0) 370 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:48.777764+0000 mgr.y (mgr.14144) 115 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:48.580713+0000 mon.a (mon.0) 367 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T08:46:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:48.580780+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-17T08:46:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:48.580957+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:48.581293+0000 mon.b (mon.2) 13 : audit [INF] from='osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:48.581695+0000 mon.a (mon.0) 370 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:46:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:48.777764+0000 mgr.y (mgr.14144) 115 : cluster [DBG] pgmap v103: 1 pgs: 1 active+clean; 0 B data, 25 MiB used, 447 GiB / 447 GiB avail 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:49.583343+0000 mon.a (mon.0) 371 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:49.583429+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:49.583938+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:49.588242+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:50.582048+0000 mon.a (mon.0) 375 : cluster [INF] osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962] boot 2024-09-17T08:46:50.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:50.582102+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-17T08:46:50.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:50.582270+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:49.583343+0000 mon.a (mon.0) 371 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:49.583429+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:49.583938+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:49.588242+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:50.582048+0000 mon.a (mon.0) 375 : cluster [INF] osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962] boot 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:50.582102+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:50.582270+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:49.583343+0000 mon.a (mon.0) 371 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:49.583429+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:49.583938+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:49.588242+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:50.582048+0000 mon.a (mon.0) 375 : cluster [INF] osd.5 [v2:172.21.15.105:6808/1140342962,v1:172.21.15.105:6809/1140342962] boot 2024-09-17T08:46:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:50.582102+0000 mon.a (mon.0) 376 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-17T08:46:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:50.582270+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:48.633616+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:48.633703+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:50.778295+0000 mgr.y (mgr.14144) 116 : cluster [DBG] pgmap v106: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:51.477018+0000 mgr.y (mgr.14144) 117 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:51.477894+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:51.479371+0000 mgr.y (mgr.14144) 118 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:51.479696+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:51.480514+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:51.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:51.583202+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-17T08:46:52.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:48.633616+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:48.633703+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:50.778295+0000 mgr.y (mgr.14144) 116 : cluster [DBG] pgmap v106: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:51.477018+0000 mgr.y (mgr.14144) 117 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:51.477894+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:51.479371+0000 mgr.y (mgr.14144) 118 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:51.479696+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:51.480514+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:51.583202+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-17T08:46:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:48.633616+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:46:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:48.633703+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:46:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:50.778295+0000 mgr.y (mgr.14144) 116 : cluster [DBG] pgmap v106: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:51.477018+0000 mgr.y (mgr.14144) 117 : audit [DBG] from='client.34247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:46:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:51.477894+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:46:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:51.479371+0000 mgr.y (mgr.14144) 118 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:46:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:51.479696+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:46:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:51.480514+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:46:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:51.583202+0000 mon.a (mon.0) 381 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-17T08:46:53.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:51.996272+0000 mgr.y (mgr.14144) 119 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:52.000914+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:52.006304+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:52.008065+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:52.012642+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:52.584297+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:51.996272+0000 mgr.y (mgr.14144) 119 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:52.000914+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:52.006304+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:52.008065+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:53.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:52.012642+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:52.584297+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:51.996272+0000 mgr.y (mgr.14144) 119 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:52.000914+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:52.006304+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:52.008065+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:52.012642+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:46:53.267 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:52.584297+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-17T08:46:54.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:52.778566+0000 mgr.y (mgr.14144) 120 : cluster [DBG] pgmap v109: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:54.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:53.794618+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.105:0/3743742892' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:53.795109+0000 mon.a (mon.0) 387 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:53.799259+0000 mon.a (mon.0) 388 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]': finished 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:53.799337+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:53.799481+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:52.778566+0000 mgr.y (mgr.14144) 120 : cluster [DBG] pgmap v109: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:53.794618+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.105:0/3743742892' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:53.795109+0000 mon.a (mon.0) 387 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:53.799259+0000 mon.a (mon.0) 388 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]': finished 2024-09-17T08:46:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:53.799337+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-17T08:46:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:53.799481+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:46:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:52.778566+0000 mgr.y (mgr.14144) 120 : cluster [DBG] pgmap v109: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:53.794618+0000 mon.b (mon.2) 14 : audit [INF] from='client.? 172.21.15.105:0/3743742892' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:53.795109+0000 mon.a (mon.0) 387 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]: dispatch 2024-09-17T08:46:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:53.799259+0000 mon.a (mon.0) 388 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ecf7dae8-bb99-4250-8e53-87d2929b6799"}]': finished 2024-09-17T08:46:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:53.799337+0000 mon.a (mon.0) 389 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-17T08:46:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:53.799481+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:46:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:54.222616+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.105:0/69033856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:55.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:54.222616+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.105:0/69033856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:55.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:54.222616+0000 mon.b (mon.2) 15 : audit [DBG] from='client.? 172.21.15.105:0/69033856' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:46:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:54.779044+0000 mgr.y (mgr.14144) 121 : cluster [DBG] pgmap v111: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:56.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:54.779044+0000 mgr.y (mgr.14144) 121 : cluster [DBG] pgmap v111: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:56.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:54.779044+0000 mgr.y (mgr.14144) 121 : cluster [DBG] pgmap v111: 1 pgs: 1 peering; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:58.685 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:56.779511+0000 mgr.y (mgr.14144) 122 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:59.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:56.779511+0000 mgr.y (mgr.14144) 122 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:59.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:56.779511+0000 mgr.y (mgr.14144) 122 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 30 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:46:59.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:59.382734+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T08:46:59.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:46:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:46:59.383562+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:00.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:59.382734+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T08:47:00.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:46:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:46:59.383562+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:59.382734+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T08:47:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:46:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:46:59.383562+0000 mon.a (mon.0) 392 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:00.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:46:58.779886+0000 mgr.y (mgr.14144) 123 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:00.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:46:59.384223+0000 mgr.y (mgr.14144) 124 : cephadm [INF] Deploying daemon osd.6 on smithi105 2024-09-17T08:47:01.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:46:58.779886+0000 mgr.y (mgr.14144) 123 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:01.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:46:59.384223+0000 mgr.y (mgr.14144) 124 : cephadm [INF] Deploying daemon osd.6 on smithi105 2024-09-17T08:47:01.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:46:58.779886+0000 mgr.y (mgr.14144) 123 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:01.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:46:59.384223+0000 mgr.y (mgr.14144) 124 : cephadm [INF] Deploying daemon osd.6 on smithi105 2024-09-17T08:47:01.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:00.780153+0000 mgr.y (mgr.14144) 125 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:02.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:00.780153+0000 mgr.y (mgr.14144) 125 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:02.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:00.780153+0000 mgr.y (mgr.14144) 125 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:02.768 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 6 on host 'smithi105' 2024-09-17T08:47:03.352 DEBUG:teuthology.orchestra.run.smithi105:osd.6> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T08:47:03.355 INFO:tasks.cephadm:Deploying osd.7 on smithi105 with /dev/vg_nvme/lv_1... 2024-09-17T08:47:03.355 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-17T08:47:03.595 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:02.756635+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:03.596 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:02.761259+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:03.596 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:02.766002+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:02.756635+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:04.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:02.761259+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:04.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:02.766002+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:02.756635+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:02.761259+0000 mon.a (mon.0) 394 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:02.766002+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:02.780378+0000 mgr.y (mgr.14144) 126 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:05.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:02.780378+0000 mgr.y (mgr.14144) 126 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:05.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:02.780378+0000 mgr.y (mgr.14144) 126 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:05.709 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-17T08:47:05.710 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-17T08:47:05.710 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-09-17T08:47:05.710 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-09-17T08:47:05.710 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129028 s, 813 MB/s 2024-09-17T08:47:05.710 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-09-17T08:47:06.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 08:47:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T08:47:05.886+0000 7ff734049080 -1 osd.6 0 log_to_monitors {default=true} 2024-09-17T08:47:06.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:04.780862+0000 mgr.y (mgr.14144) 127 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:06.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:05.458908+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:05.890362+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:05.890762+0000 mon.a (mon.0) 397 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:04.780862+0000 mgr.y (mgr.14144) 127 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:05.458908+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:05.890362+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:05.890762+0000 mon.a (mon.0) 397 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:06.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:04.780862+0000 mgr.y (mgr.14144) 127 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:05.458908+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:05.890362+0000 mon.b (mon.2) 16 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:05.890762+0000 mon.a (mon.0) 397 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T08:47:07.317 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi105:vg_nvme/lv_1 2024-09-17T08:47:07.904 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T08:47:07.585+0000 7ff72a5ba700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T08:47:07.905 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T08:47:07.593+0000 7ff726d36700 -1 osd.6 41 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-09-17T08:47:07.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:06.582110+0000 mon.a (mon.0) 398 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T08:47:07.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:06.582209+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-17T08:47:07.905 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:06.582374+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:07.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:06.582734+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:07.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:06.583155+0000 mon.a (mon.0) 401 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:07.906 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:06.781308+0000 mgr.y (mgr.14144) 128 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:06.582110+0000 mon.a (mon.0) 398 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T08:47:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:06.582209+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-17T08:47:08.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:06.582374+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:06.582734+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:06.583155+0000 mon.a (mon.0) 401 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:06.781308+0000 mgr.y (mgr.14144) 128 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:06.582110+0000 mon.a (mon.0) 398 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:06.582209+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:06.582374+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:06.582734+0000 mon.b (mon.2) 17 : audit [INF] from='osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:06.583155+0000 mon.a (mon.0) 401 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:06.781308+0000 mgr.y (mgr.14144) 128 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 31 MiB used, 536 GiB / 536 GiB avail 2024-09-17T08:47:08.705 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:07.583506+0000 mon.a (mon.0) 402 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:07.583623+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:07.584333+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:07.587239+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:08.588561+0000 mon.a (mon.0) 406 : cluster [INF] osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313] boot 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:08.588722+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-17T08:47:09.151 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:08.588955+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:07.583506+0000 mon.a (mon.0) 402 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:07.583623+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:07.584333+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:07.587239+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:08.588561+0000 mon.a (mon.0) 406 : cluster [INF] osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313] boot 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:08.588722+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-17T08:47:09.152 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:08.588955+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:07.583506+0000 mon.a (mon.0) 402 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:07.583623+0000 mon.a (mon.0) 403 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:07.584333+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:07.587239+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:08.588561+0000 mon.a (mon.0) 406 : cluster [INF] osd.6 [v2:172.21.15.105:6816/2946684313,v1:172.21.15.105:6817/2946684313] boot 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:08.588722+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-17T08:47:09.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:08.588955+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:47:09.719 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:06.919287+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:06.919381+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:08.781656+0000 mgr.y (mgr.14144) 129 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:09.519606+0000 mgr.y (mgr.14144) 130 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:09.524324+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:09.529076+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:09.530789+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:09.535465+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:09.720 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:09.586559+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:06.919287+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:06.919381+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:08.781656+0000 mgr.y (mgr.14144) 129 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:09.519606+0000 mgr.y (mgr.14144) 130 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:09.524324+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:09.529076+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:09.530789+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:09.535465+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:09.586559+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:06.919287+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:06.919381+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:08.781656+0000 mgr.y (mgr.14144) 129 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:09.519606+0000 mgr.y (mgr.14144) 130 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:09.524324+0000 mon.a (mon.0) 409 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:09.529076+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:09.530789+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:09.535465+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:10.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:09.586559+0000 mon.a (mon.0) 413 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-17T08:47:10.701 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.139053+0000 mgr.y (mgr.14144) 131 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:10.702 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.140023+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:47:10.702 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:10.141428+0000 mgr.y (mgr.14144) 132 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:47:10.702 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.141735+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:47:10.702 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.142551+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:10.702 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:10.589115+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-17T08:47:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.139053+0000 mgr.y (mgr.14144) 131 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.140023+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:47:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:10.141428+0000 mgr.y (mgr.14144) 132 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:47:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.141735+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.142551+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:10.589115+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.139053+0000 mgr.y (mgr.14144) 131 : audit [DBG] from='client.34274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.140023+0000 mon.a (mon.0) 414 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:10.141428+0000 mgr.y (mgr.14144) 132 : cephadm [INF] Applying service osd.None on host smithi105... 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.141735+0000 mon.a (mon.0) 415 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.142551+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:10.589115+0000 mon.a (mon.0) 417 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-17T08:47:11.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:10.781981+0000 mgr.y (mgr.14144) 133 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:11.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.883069+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:47:11.666 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:10.889500+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:47:12.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:10.781981+0000 mgr.y (mgr.14144) 133 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:12.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.883069+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:47:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:10.889500+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:47:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:10.781981+0000 mgr.y (mgr.14144) 133 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.883069+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:47:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:10.889500+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:12.275447+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.105:0/4137398815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:12.275871+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:12.280603+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]': finished 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:12.280698+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:12.280850+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:12.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:12.275447+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.105:0/4137398815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:12.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:12.275871+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:12.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:12.280603+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]': finished 2024-09-17T08:47:12.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:12.280698+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-17T08:47:12.910 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:12.280850+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:13.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:12.275447+0000 mon.b (mon.2) 18 : audit [INF] from='client.? 172.21.15.105:0/4137398815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:12.275871+0000 mon.a (mon.0) 420 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]: dispatch 2024-09-17T08:47:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:12.280603+0000 mon.a (mon.0) 421 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169"}]': finished 2024-09-17T08:47:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:12.280698+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-17T08:47:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:12.280850+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:14.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:12.693332+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.105:0/1892162046' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:47:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:12.782518+0000 mgr.y (mgr.14144) 134 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:12.693332+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.105:0/1892162046' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:47:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:12.782518+0000 mgr.y (mgr.14144) 134 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:12.693332+0000 mon.b (mon.2) 19 : audit [DBG] from='client.? 172.21.15.105:0/1892162046' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T08:47:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:12.782518+0000 mgr.y (mgr.14144) 134 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:16.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:14.782828+0000 mgr.y (mgr.14144) 135 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:14.782828+0000 mgr.y (mgr.14144) 135 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:16.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:14.782828+0000 mgr.y (mgr.14144) 135 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:18.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:16.783289+0000 mgr.y (mgr.14144) 136 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:17.881437+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T08:47:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:17.882283+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:19.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:16.783289+0000 mgr.y (mgr.14144) 136 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:17.881437+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T08:47:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:17.882283+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:16.783289+0000 mgr.y (mgr.14144) 136 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:17.881437+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T08:47:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:17.882283+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:47:19.761 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:17.883014+0000 mgr.y (mgr.14144) 137 : cephadm [INF] Deploying daemon osd.7 on smithi105 2024-09-17T08:47:19.761 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:18.783594+0000 mgr.y (mgr.14144) 138 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:20.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:17.883014+0000 mgr.y (mgr.14144) 137 : cephadm [INF] Deploying daemon osd.7 on smithi105 2024-09-17T08:47:20.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:18.783594+0000 mgr.y (mgr.14144) 138 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:20.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:17.883014+0000 mgr.y (mgr.14144) 137 : cephadm [INF] Deploying daemon osd.7 on smithi105 2024-09-17T08:47:20.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:18.783594+0000 mgr.y (mgr.14144) 138 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:21.198 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 7 on host 'smithi105' 2024-09-17T08:47:21.863 DEBUG:teuthology.orchestra.run.smithi105:osd.7> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T08:47:21.865 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T08:47:21.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T08:47:22.152 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:20.784007+0000 mgr.y (mgr.14144) 139 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:22.153 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:21.186911+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:22.153 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:21.191349+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:22.153 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:21.196207+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:22.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:20.784007+0000 mgr.y (mgr.14144) 139 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:22.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:21.186911+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:22.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:21.191349+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:22.180 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:21.196207+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:22.180 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:20.784007+0000 mgr.y (mgr.14144) 139 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:22.181 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:21.186911+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:22.181 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:21.191349+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:22.181 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:21.196207+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:23.942 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:24.166 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:22.784526+0000 mgr.y (mgr.14144) 140 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:24.166 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:22.862344+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:22.784526+0000 mgr.y (mgr.14144) 140 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:24.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:22.862344+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:22.784526+0000 mgr.y (mgr.14144) 140 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:24.230 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:22.862344+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:24.487 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 08:47:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T08:47:24.371+0000 7fd6f29da080 -1 osd.7 0 log_to_monitors {default=true} 2024-09-17T08:47:24.546 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":45,"num_osds":8,"num_up_osds":7,"osd_up_since":1726562828,"num_in_osds":8,"osd_in_since":1726562832,"num_remapped_pgs":0} 2024-09-17T08:47:25.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:23.941809+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.27:0/1335582005' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:25.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.374613+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.164 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.375118+0000 mon.a (mon.0) 430 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:23.941809+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.27:0/1335582005' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:25.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.374613+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.375118+0000 mon.a (mon.0) 430 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:23.941809+0000 mon.c (mon.1) 9 : audit [DBG] from='client.? 172.21.15.27:0/1335582005' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.374613+0000 mon.b (mon.2) 20 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.375118+0000 mon.a (mon.0) 430 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T08:47:25.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T08:47:26.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:24.784797+0000 mgr.y (mgr.14144) 141 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:26.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.869590+0000 mon.a (mon.0) 431 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T08:47:26.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:24.869682+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-17T08:47:26.193 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.869962+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.870205+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:24.870554+0000 mon.a (mon.0) 434 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:24.784797+0000 mgr.y (mgr.14144) 141 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.869590+0000 mon.a (mon.0) 431 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:24.869682+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.869962+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.870205+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:26.194 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:24.870554+0000 mon.a (mon.0) 434 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:26.277 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T08:47:25.878+0000 7fd6e8f4b700 -1 osd.7 0 waiting for initial osdmap 2024-09-17T08:47:26.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T08:47:25.886+0000 7fd6e36c3700 -1 osd.7 47 set_numa_affinity unable to identify public interface 'enp3s0f1' numa node: (0) Success 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:24.784797+0000 mgr.y (mgr.14144) 141 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.869590+0000 mon.a (mon.0) 431 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:24.869682+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.869962+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.870205+0000 mon.b (mon.2) 21 : audit [INF] from='osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:24.870554+0000 mon.a (mon.0) 434 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T08:47:27.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.869990+0000 mon.a (mon.0) 435 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:25.870068+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.870635+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.879211+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:25.909715+0000 mgr.y (mgr.14144) 142 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.916067+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.919232+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.920491+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:25.924277+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.869990+0000 mon.a (mon.0) 435 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:27.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:25.870068+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.870635+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.879211+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:25.909715+0000 mgr.y (mgr.14144) 142 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.916067+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.919232+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.920491+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:27.015 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:25.924277+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.869990+0000 mon.a (mon.0) 435 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:25.870068+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.870635+0000 mon.a (mon.0) 437 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.879211+0000 mon.a (mon.0) 438 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:25.909715+0000 mgr.y (mgr.14144) 142 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.916067+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.919232+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.920491+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:25.924277+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:27.401 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:28.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:25.372675+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:25.372874+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:26.785098+0000 mgr.y (mgr.14144) 143 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:26.880885+0000 mon.a (mon.0) 443 : cluster [INF] osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433] boot 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:26.881067+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:26.881558+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:27.401755+0000 mon.c (mon.1) 10 : audit [DBG] from='client.? 172.21.15.27:0/447105794' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:27.579756+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:25.372675+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:25.372874+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:28.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:26.785098+0000 mgr.y (mgr.14144) 143 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:28.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:26.880885+0000 mon.a (mon.0) 443 : cluster [INF] osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433] boot 2024-09-17T08:47:28.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:26.881067+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-17T08:47:28.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:26.881558+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:28.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:27.401755+0000 mon.c (mon.1) 10 : audit [DBG] from='client.? 172.21.15.27:0/447105794' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:28.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:27.579756+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-17T08:47:28.052 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":48,"num_osds":8,"num_up_osds":8,"osd_up_since":1726562846,"num_in_osds":8,"osd_in_since":1726562832,"num_remapped_pgs":1} 2024-09-17T08:47:28.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T08:47:28.255 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:47:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:25.372675+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:25.372874+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:26.785098+0000 mgr.y (mgr.14144) 143 : cluster [DBG] pgmap v135: 1 pgs: 1 active+clean; 0 B data, 36 MiB used, 626 GiB / 626 GiB avail 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:26.880885+0000 mon.a (mon.0) 443 : cluster [INF] osd.7 [v2:172.21.15.105:6824/3306013433,v1:172.21.15.105:6825/3306013433] boot 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:26.881067+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:26.881558+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:27.401755+0000 mon.c (mon.1) 10 : audit [DBG] from='client.? 172.21.15.27:0/447105794' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T08:47:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:27.579756+0000 mon.a (mon.0) 446 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-17T08:47:29.949 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:29.949 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":50,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","created":"2024-09-17T08:43:19.989471+0000","modified":"2024-09-17T08:47:28.583482+0000","last_up_change":"2024-09-17T08:47:26.876081+0000","last_in_change":"2024-09-17T08:47:12.276467+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-09-17T08:45:57.543876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"46e1cb43-f6d6-4bc3-bc29-7067084ed835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6803","nonce":1678887019}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6805","nonce":1678887019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6809","nonce":1678887019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6807","nonce":1678887019}]},"public_addr":"172.21.15.27:6803/1678887019","cluster_addr":"172.21.15.27:6805/1678887019","heartbeat_back_addr":"172.21.15.27:6809/1678887019","heartbeat_front_addr":"172.21.15.27:6807/1678887019","state":["exists","up"]},{"osd":1,"uuid":"40dcd331-2a54-4ef7-8d4e-5117cbca6438","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6811","nonce":2462012426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6813","nonce":2462012426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6817","nonce":2462012426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6815","nonce":2462012426}]},"public_addr":"172.21.15.27:6811/2462012426","cluster_addr":"172.21.15.27:6813/2462012426","heartbeat_back_addr":"172.21.15.27:6817/2462012426","heartbeat_front_addr":"172.21.15.27:6815/2462012426","state":["exists","up"]},{"osd":2,"uuid":"09227eb6-9bec-49b0-a1e8-a954c1872265","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6819","nonce":3166119382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6821","nonce":3166119382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6825","nonce":3166119382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6823","nonce":3166119382}]},"public_addr":"172.21.15.27:6819/3166119382","cluster_addr":"172.21.15.27:6821/3166119382","heartbeat_back_addr":"172.21.15.27:6825/3166119382","heartbeat_front_addr":"172.21.15.27:6823/3166119382","state":["exists","up"]},{"osd":3,"uuid":"9e66dcd4-23e5-42b7-b665-8fa128130229","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6827","nonce":1975358975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6829","nonce":1975358975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6833","nonce":1975358975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6831","nonce":1975358975}]},"public_addr":"172.21.15.27:6827/1975358975","cluster_addr":"172.21.15.27:6829/1975358975","heartbeat_back_addr":"172.21.15.27:6833/1975358975","heartbeat_front_addr":"172.21.15.27:6831/1975358975","state":["exists","up"]},{"osd":4,"uuid":"34b2e281-2987-41a2-adc6-fb895ded7659","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6801","nonce":4256538191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6803","nonce":4256538191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6807","nonce":4256538191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6805","nonce":4256538191}]},"public_addr":"172.21.15.105:6801/4256538191","cluster_addr":"172.21.15.105:6803/4256538191","heartbeat_back_addr":"172.21.15.105:6807/4256538191","heartbeat_front_addr":"172.21.15.105:6805/4256538191","state":["exists","up"]},{"osd":5,"uuid":"73eb3b08-0c61-45b2-a7c2-ceac17ef7982","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6809","nonce":1140342962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6811","nonce":1140342962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6815","nonce":1140342962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6813","nonce":1140342962}]},"public_addr":"172.21.15.105:6809/1140342962","cluster_addr":"172.21.15.105:6811/1140342962","heartbeat_back_addr":"172.21.15.105:6815/1140342962","heartbeat_front_addr":"172.21.15.105:6813/1140342962","state":["exists","up"]},{"osd":6,"uuid":"ecf7dae8-bb99-4250-8e53-87d2929b6799","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6817","nonce":2946684313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6819","nonce":2946684313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6823","nonce":2946684313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6821","nonce":2946684313}]},"public_addr":"172.21.15.105:6817/2946684313","cluster_addr":"172.21.15.105:6819/2946684313","heartbeat_back_addr":"172.21.15.105:6823/2946684313","heartbeat_front_addr":"172.21.15.105:6821/2946684313","state":["exists","up"]},{"osd":7,"uuid":"e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6825","nonce":3306013433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6827","nonce":3306013433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6831","nonce":3306013433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6829","nonce":3306013433}]},"public_addr":"172.21.15.105:6825/3306013433","cluster_addr":"172.21.15.105:6827/3306013433","heartbeat_back_addr":"172.21.15.105:6831/3306013433","heartbeat_front_addr":"172.21.15.105:6829/3306013433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:17.322071+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:35.898970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:55.114365+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:15.160089+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:30.830357+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:48.633705+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:06.919383+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:25.372879+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/466285564":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6800/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:0/1001596001":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/277636790":"2024-09-18T08:43:51.105114+0000","172.21.15.27:6800/3868462814":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/3348792656":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3868462814":"2024-09-18T08:43:51.105114+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T08:47:29.950 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:28.590256+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-17T08:47:29.950 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:28.785452+0000 mgr.y (mgr.14144) 144 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:29.950 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:28.590256+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-17T08:47:29.950 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:28.785452+0000 mgr.y (mgr.14144) 144 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:30.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:28.590256+0000 mon.a (mon.0) 447 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-17T08:47:30.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:28.785452+0000 mgr.y (mgr.14144) 144 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:30.577 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2024-09-17T08:45:57.543876+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2024-09-17T08:47:30.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd pool get device_health_metrics pg_num 2024-09-17T08:47:30.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:29.948339+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.27:0/125257618' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:30.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:29.948339+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.27:0/125257618' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:30.783 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:47:31.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:29.948339+0000 mon.c (mon.1) 11 : audit [DBG] from='client.? 172.21.15.27:0/125257618' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:31.842 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:30.785723+0000 mgr.y (mgr.14144) 145 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:31.842 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:30.785723+0000 mgr.y (mgr.14144) 145 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:32.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:30.785723+0000 mgr.y (mgr.14144) 145 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:32.504 INFO:teuthology.orchestra.run.smithi027.stdout:pg_num: 1 2024-09-17T08:47:32.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:32.504258+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.27:0/1653269240' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-09-17T08:47:32.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:32.504258+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.27:0/1653269240' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-09-17T08:47:33.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:32.504258+0000 mon.c (mon.1) 12 : audit [DBG] from='client.? 172.21.15.27:0/1653269240' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2024-09-17T08:47:33.123 INFO:tasks.cephadm:Adding prometheus.a on smithi105 2024-09-17T08:47:33.123 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi105=a' 2024-09-17T08:47:33.750 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:32.786221+0000 mgr.y (mgr.14144) 146 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:32.786221+0000 mgr.y (mgr.14144) 146 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:34.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:32.786221+0000 mgr.y (mgr.14144) 146 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:34.948 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled prometheus update... 2024-09-17T08:47:35.522 DEBUG:teuthology.orchestra.run.smithi105:prometheus.a> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-17T08:47:35.524 INFO:tasks.cephadm:Adding node-exporter.a on smithi027 2024-09-17T08:47:35.525 INFO:tasks.cephadm:Adding node-exporter.b on smithi105 2024-09-17T08:47:35.525 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi027=a;smithi105=b' 2024-09-17T08:47:36.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:34.786645+0000 mgr.y (mgr.14144) 147 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:36.179 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:34.942964+0000 mgr.y (mgr.14144) 148 : audit [DBG] from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:36.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:34.943745+0000 mgr.y (mgr.14144) 149 : cephadm [INF] Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T08:47:36.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:34.947992+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:34.948840+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:36.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:35.182485+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:35.191716+0000 mgr.y (mgr.14144) 150 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:34.786645+0000 mgr.y (mgr.14144) 147 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:34.942964+0000 mgr.y (mgr.14144) 148 : audit [DBG] from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:34.943745+0000 mgr.y (mgr.14144) 149 : cephadm [INF] Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:34.947992+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:34.948840+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:35.182485+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:35.191716+0000 mgr.y (mgr.14144) 150 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:34.786645+0000 mgr.y (mgr.14144) 147 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:34.942964+0000 mgr.y (mgr.14144) 148 : audit [DBG] from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:36.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:34.943745+0000 mgr.y (mgr.14144) 149 : cephadm [INF] Saving service prometheus spec with placement smithi105=a;count:1 2024-09-17T08:47:36.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:34.947992+0000 mon.a (mon.0) 448 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:34.948840+0000 mon.a (mon.0) 449 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:36.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:35.182485+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:36.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:35.191716+0000 mgr.y (mgr.14144) 150 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T08:47:37.440 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled node-exporter update... 2024-09-17T08:47:38.095 DEBUG:teuthology.orchestra.run.smithi027:node-exporter.a> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-17T08:47:38.099 DEBUG:teuthology.orchestra.run.smithi105:node-exporter.b> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-17T08:47:38.101 INFO:tasks.cephadm:Adding alertmanager.a on smithi027 2024-09-17T08:47:38.101 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi027=a' 2024-09-17T08:47:38.733 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:36.787082+0000 mgr.y (mgr.14144) 151 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:38.733 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:37.434955+0000 mgr.y (mgr.14144) 152 : audit [DBG] from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:38.734 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:37.435763+0000 mgr.y (mgr.14144) 153 : cephadm [INF] Saving service node-exporter spec with placement smithi027=a;smithi105=b;count:2 2024-09-17T08:47:38.734 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:37.439728+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:38.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:36.787082+0000 mgr.y (mgr.14144) 151 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:38.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:37.434955+0000 mgr.y (mgr.14144) 152 : audit [DBG] from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:38.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:37.435763+0000 mgr.y (mgr.14144) 153 : cephadm [INF] Saving service node-exporter spec with placement smithi027=a;smithi105=b;count:2 2024-09-17T08:47:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:37.439728+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:36.787082+0000 mgr.y (mgr.14144) 151 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:37.434955+0000 mgr.y (mgr.14144) 152 : audit [DBG] from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi027=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:37.435763+0000 mgr.y (mgr.14144) 153 : cephadm [INF] Saving service node-exporter spec with placement smithi027=a;smithi105=b;count:2 2024-09-17T08:47:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:37.439728+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:39.977 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled alertmanager update... 2024-09-17T08:47:40.583 DEBUG:teuthology.orchestra.run.smithi027:alertmanager.a> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-17T08:47:40.586 INFO:tasks.cephadm:Adding grafana.a on smithi105 2024-09-17T08:47:40.586 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi105=a' 2024-09-17T08:47:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:38.787364+0000 mgr.y (mgr.14144) 154 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:39.976748+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:40.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:38.787364+0000 mgr.y (mgr.14144) 154 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:40.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:39.976748+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:40.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:38.787364+0000 mgr.y (mgr.14144) 154 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:40.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:39.976748+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:41.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:39.971346+0000 mgr.y (mgr.14144) 155 : audit [DBG] from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:39.972123+0000 mgr.y (mgr.14144) 156 : cephadm [INF] Saving service alertmanager spec with placement smithi027=a;count:1 2024-09-17T08:47:41.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:39.971346+0000 mgr.y (mgr.14144) 155 : audit [DBG] from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:41.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:39.972123+0000 mgr.y (mgr.14144) 156 : cephadm [INF] Saving service alertmanager spec with placement smithi027=a;count:1 2024-09-17T08:47:41.758 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:39.971346+0000 mgr.y (mgr.14144) 155 : audit [DBG] from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:41.758 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:39.972123+0000 mgr.y (mgr.14144) 156 : cephadm [INF] Saving service alertmanager spec with placement smithi027=a;count:1 2024-09-17T08:47:42.424 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled grafana update... 2024-09-17T08:47:42.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:40.787738+0000 mgr.y (mgr.14144) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:42.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:42.423313+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:40.787738+0000 mgr.y (mgr.14144) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:42.423313+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:42.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:40.787738+0000 mgr.y (mgr.14144) 157 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:42.423313+0000 mon.a (mon.0) 453 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:43.414 DEBUG:teuthology.orchestra.run.smithi105:grafana.a> sudo journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service 2024-09-17T08:47:43.416 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T08:47:43.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T08:47:43.731 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:42.418608+0000 mgr.y (mgr.14144) 158 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:43.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:42.419358+0000 mgr.y (mgr.14144) 159 : cephadm [INF] Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T08:47:43.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:42.418608+0000 mgr.y (mgr.14144) 158 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:43.732 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:42.419358+0000 mgr.y (mgr.14144) 159 : cephadm [INF] Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T08:47:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:42.418608+0000 mgr.y (mgr.14144) 158 : audit [DBG] from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:47:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:42.419358+0000 mgr.y (mgr.14144) 159 : cephadm [INF] Saving service grafana spec with placement smithi105=a;count:1 2024-09-17T08:47:44.756 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:42.788116+0000 mgr.y (mgr.14144) 160 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:44.860 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:42.788116+0000 mgr.y (mgr.14144) 160 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:44.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:42.788116+0000 mgr.y (mgr.14144) 160 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:45.473 INFO:teuthology.orchestra.run.smithi027.stdout:[client.0] 2024-09-17T08:47:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: key = AQAxQulm0PT1GxAATkMUc6BYEbvqjwf+7GiOrg== 2024-09-17T08:47:46.209 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2024-09-17T08:47:46.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T08:47:46.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T08:47:46.253 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T08:47:46.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:44.788379+0000 mgr.y (mgr.14144) 161 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:46.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:45.468660+0000 mon.c (mon.1) 13 : audit [INF] from='client.? 172.21.15.27:0/368550893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:45.468970+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:45.472298+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:46.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:44.788379+0000 mgr.y (mgr.14144) 161 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:46.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:45.468660+0000 mon.c (mon.1) 13 : audit [INF] from='client.? 172.21.15.27:0/368550893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:45.468970+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:45.472298+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:46.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:44.788379+0000 mgr.y (mgr.14144) 161 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:46.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:45.468660+0000 mon.c (mon.1) 13 : audit [INF] from='client.? 172.21.15.27:0/368550893' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:45.468970+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:46.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:45.472298+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:48.644 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:46.788844+0000 mgr.y (mgr.14144) 162 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:48.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:46.788844+0000 mgr.y (mgr.14144) 162 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:48.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:46.788844+0000 mgr.y (mgr.14144) 162 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:50.733 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:48.789151+0000 mgr.y (mgr.14144) 163 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:50.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:48.789151+0000 mgr.y (mgr.14144) 163 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:50.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:48.789151+0000 mgr.y (mgr.14144) 163 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:51.568 INFO:teuthology.orchestra.run.smithi105.stdout:[client.1] 2024-09-17T08:47:51.568 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQA3QulmZ1yaIRAAkbUo7uvSb9ZNCoVDsCr7CA== 2024-09-17T08:47:52.189 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-17T08:47:52.190 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-17T08:47:52.190 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-17T08:47:52.228 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T08:47:52.228 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T08:47:52.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T08:47:52.445 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:47:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:50.789655+0000 mgr.y (mgr.14144) 164 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:51.563046+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.105:0/2948351393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:51.563595+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:51.567023+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:52.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:50.789655+0000 mgr.y (mgr.14144) 164 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:52.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:51.563046+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.105:0/2948351393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:51.563595+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:51.567023+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:52.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:50.789655+0000 mgr.y (mgr.14144) 164 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:52.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:51.563046+0000 mon.b (mon.2) 22 : audit [INF] from='client.? 172.21.15.105:0/2948351393' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:51.563595+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T08:47:52.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:51.567023+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T08:47:53.635 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:47:53.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:52.789931+0000 mgr.y (mgr.14144) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:53.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:52.789931+0000 mgr.y (mgr.14144) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:53.910 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:52.789931+0000 mgr.y (mgr.14144) 165 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:53.910 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 podman[35635]: 2024-09-17 08:47:53.575128006 +0000 UTC m=+0.016665239 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:47:53.910 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 podman[35635]: 2024-09-17 08:47:53.681970369 +0000 UTC m=+0.123507627 container create cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:47:53.910 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 podman[35635]: 2024-09-17 08:47:53.910480499 +0000 UTC m=+0.352017713 container init cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:47:54.207 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 podman[35635]: 2024-09-17 08:47:53.913260262 +0000 UTC m=+0.354797475 container start cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.943Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.943Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.943Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.943Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.944Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.944Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.944Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:47:54.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.945Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.947Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.948Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.948Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=119.199µs 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.948Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.948Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.948Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.949Z caller=main.go:827 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:47:53.950Z caller=main.go:646 msg="Server is ready to receive web requests." 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:54 smithi105 bash[35635]: cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 2024-09-17T08:47:54.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:47:54 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:47:54.835 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":14,"active_gid":14144,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":4197914178},{"type":"v1","addr":"172.21.15.27:6801","nonce":4197914178}]},"active_addr":"172.21.15.27:6801/4197914178","active_change":"2024-09-17T08:44:10.699821+0000","active_mgr_features":4540138297136906239,"available":true,"standbys":[{"gid":34109,"name":"x","mgr_features":4540138297136906239,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"haproxy","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/alertmanager:v0.20.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"docker.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"docker.io/ceph/ceph-grafana:6.7.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"haproxy","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/node-exporter:v0.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"docker.io/prom/prometheus:v2.18.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository url","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_cephfs":{"name":"FEATURE_TOGGLE_cephfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_iscsi":{"name":"FEATURE_TOGGLE_iscsi","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_mirroring":{"name":"FEATURE_TOGGLE_mirroring","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_nfs":{"name":"FEATURE_TOGGLE_nfs","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rbd":{"name":"FEATURE_TOGGLE_rbd","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_rgw":{"name":"FEATURE_TOGGLE_rgw","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_HOST":{"name":"RGW_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_PORT":{"name":"RGW_API_PORT","type":"str","level":"advanced","flags":0,"default_value":"80","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SCHEME":{"name":"RGW_API_SCHEME","type":"str","level":"advanced","flags":0,"default_value":"http","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_USER_ID":{"name":"RGW_API_USER_ID","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"persist_interval":{"name":"persist_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how frequently to persist completed events","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://smithi027.front.sepia.ceph.com:8443/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":3,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":400907835}]},{"addrvec":[{"type":"v2","addr":"172.21.15.27:0","nonce":2746046817}]}]}} 2024-09-17T08:47:54.950 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T08:47:54.950 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T08:47:54.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T08:47:55.160 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:47:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:54.067249+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:54.067978+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:55.320 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:54.203240+0000 mon.a (mon.0) 460 : audit [DBG] from='client.? 172.21.15.27:0/4284599234' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:47:55.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:54.067249+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:55.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:54.067978+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:55.321 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:54.203240+0000 mon.a (mon.0) 460 : audit [DBG] from='client.? 172.21.15.27:0/4284599234' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:47:55.452 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:54.067249+0000 mon.a (mon.0) 458 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:55.453 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:54.067978+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:47:55.453 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:54.203240+0000 mon.a (mon.0) 460 : audit [DBG] from='client.? 172.21.15.27:0/4284599234' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:47:56.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:54.790209+0000 mgr.y (mgr.14144) 166 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:56.388 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:54.790209+0000 mgr.y (mgr.14144) 166 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:56.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:54.790209+0000 mgr.y (mgr.14144) 166 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:56.974 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:56.975 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":50,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","created":"2024-09-17T08:43:19.989471+0000","modified":"2024-09-17T08:47:28.583482+0000","last_up_change":"2024-09-17T08:47:26.876081+0000","last_in_change":"2024-09-17T08:47:12.276467+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-09-17T08:45:57.543876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"46e1cb43-f6d6-4bc3-bc29-7067084ed835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6803","nonce":1678887019}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6805","nonce":1678887019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6809","nonce":1678887019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6807","nonce":1678887019}]},"public_addr":"172.21.15.27:6803/1678887019","cluster_addr":"172.21.15.27:6805/1678887019","heartbeat_back_addr":"172.21.15.27:6809/1678887019","heartbeat_front_addr":"172.21.15.27:6807/1678887019","state":["exists","up"]},{"osd":1,"uuid":"40dcd331-2a54-4ef7-8d4e-5117cbca6438","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6811","nonce":2462012426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6813","nonce":2462012426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6817","nonce":2462012426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6815","nonce":2462012426}]},"public_addr":"172.21.15.27:6811/2462012426","cluster_addr":"172.21.15.27:6813/2462012426","heartbeat_back_addr":"172.21.15.27:6817/2462012426","heartbeat_front_addr":"172.21.15.27:6815/2462012426","state":["exists","up"]},{"osd":2,"uuid":"09227eb6-9bec-49b0-a1e8-a954c1872265","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6819","nonce":3166119382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6821","nonce":3166119382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6825","nonce":3166119382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6823","nonce":3166119382}]},"public_addr":"172.21.15.27:6819/3166119382","cluster_addr":"172.21.15.27:6821/3166119382","heartbeat_back_addr":"172.21.15.27:6825/3166119382","heartbeat_front_addr":"172.21.15.27:6823/3166119382","state":["exists","up"]},{"osd":3,"uuid":"9e66dcd4-23e5-42b7-b665-8fa128130229","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6827","nonce":1975358975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6829","nonce":1975358975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6833","nonce":1975358975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6831","nonce":1975358975}]},"public_addr":"172.21.15.27:6827/1975358975","cluster_addr":"172.21.15.27:6829/1975358975","heartbeat_back_addr":"172.21.15.27:6833/1975358975","heartbeat_front_addr":"172.21.15.27:6831/1975358975","state":["exists","up"]},{"osd":4,"uuid":"34b2e281-2987-41a2-adc6-fb895ded7659","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6801","nonce":4256538191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6803","nonce":4256538191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6807","nonce":4256538191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6805","nonce":4256538191}]},"public_addr":"172.21.15.105:6801/4256538191","cluster_addr":"172.21.15.105:6803/4256538191","heartbeat_back_addr":"172.21.15.105:6807/4256538191","heartbeat_front_addr":"172.21.15.105:6805/4256538191","state":["exists","up"]},{"osd":5,"uuid":"73eb3b08-0c61-45b2-a7c2-ceac17ef7982","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6809","nonce":1140342962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6811","nonce":1140342962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6815","nonce":1140342962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6813","nonce":1140342962}]},"public_addr":"172.21.15.105:6809/1140342962","cluster_addr":"172.21.15.105:6811/1140342962","heartbeat_back_addr":"172.21.15.105:6815/1140342962","heartbeat_front_addr":"172.21.15.105:6813/1140342962","state":["exists","up"]},{"osd":6,"uuid":"ecf7dae8-bb99-4250-8e53-87d2929b6799","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6817","nonce":2946684313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6819","nonce":2946684313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6823","nonce":2946684313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6821","nonce":2946684313}]},"public_addr":"172.21.15.105:6817/2946684313","cluster_addr":"172.21.15.105:6819/2946684313","heartbeat_back_addr":"172.21.15.105:6823/2946684313","heartbeat_front_addr":"172.21.15.105:6821/2946684313","state":["exists","up"]},{"osd":7,"uuid":"e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6825","nonce":3306013433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6827","nonce":3306013433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6831","nonce":3306013433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6829","nonce":3306013433}]},"public_addr":"172.21.15.105:6825/3306013433","cluster_addr":"172.21.15.105:6827/3306013433","heartbeat_back_addr":"172.21.15.105:6831/3306013433","heartbeat_front_addr":"172.21.15.105:6829/3306013433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:17.322071+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:35.898970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:55.114365+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:15.160089+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:30.830357+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:48.633705+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:06.919383+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:25.372879+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/466285564":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6800/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:0/1001596001":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/277636790":"2024-09-18T08:43:51.105114+0000","172.21.15.27:6800/3868462814":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/3348792656":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3868462814":"2024-09-18T08:43:51.105114+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T08:47:57.640 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T08:47:57.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T08:47:57.863 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:56.612288+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:56.616255+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:47:56.617805+0000 mgr.y (mgr.14144) 167 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:56.790475+0000 mgr.y (mgr.14144) 168 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:56.973650+0000 mon.a (mon.0) 463 : audit [DBG] from='client.? 172.21.15.27:0/3180972429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:56.612288+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:56.616255+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:47:56.617805+0000 mgr.y (mgr.14144) 167 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T08:47:57.864 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:56.790475+0000 mgr.y (mgr.14144) 168 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:57.865 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:56.973650+0000 mon.a (mon.0) 463 : audit [DBG] from='client.? 172.21.15.27:0/3180972429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:57.865 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:47:58.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:56.612288+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:58.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:56.616255+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:47:58.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:47:56.617805+0000 mgr.y (mgr.14144) 167 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T08:47:58.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:56.790475+0000 mgr.y (mgr.14144) 168 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:47:58.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:56.973650+0000 mon.a (mon.0) 463 : audit [DBG] from='client.? 172.21.15.27:0/3180972429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:47:59.439 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:47:59.439 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":50,"fsid":"d61dd7c6-74d0-11ef-bceb-c7b262605968","created":"2024-09-17T08:43:19.989471+0000","modified":"2024-09-17T08:47:28.583482+0000","last_up_change":"2024-09-17T08:47:26.876081+0000","last_in_change":"2024-09-17T08:47:12.276467+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2024-09-17T08:45:57.543876+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"46e1cb43-f6d6-4bc3-bc29-7067084ed835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6803","nonce":1678887019}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6805","nonce":1678887019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6809","nonce":1678887019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":1678887019},{"type":"v1","addr":"172.21.15.27:6807","nonce":1678887019}]},"public_addr":"172.21.15.27:6803/1678887019","cluster_addr":"172.21.15.27:6805/1678887019","heartbeat_back_addr":"172.21.15.27:6809/1678887019","heartbeat_front_addr":"172.21.15.27:6807/1678887019","state":["exists","up"]},{"osd":1,"uuid":"40dcd331-2a54-4ef7-8d4e-5117cbca6438","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6811","nonce":2462012426}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6813","nonce":2462012426}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6817","nonce":2462012426}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":2462012426},{"type":"v1","addr":"172.21.15.27:6815","nonce":2462012426}]},"public_addr":"172.21.15.27:6811/2462012426","cluster_addr":"172.21.15.27:6813/2462012426","heartbeat_back_addr":"172.21.15.27:6817/2462012426","heartbeat_front_addr":"172.21.15.27:6815/2462012426","state":["exists","up"]},{"osd":2,"uuid":"09227eb6-9bec-49b0-a1e8-a954c1872265","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6819","nonce":3166119382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6821","nonce":3166119382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6825","nonce":3166119382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":3166119382},{"type":"v1","addr":"172.21.15.27:6823","nonce":3166119382}]},"public_addr":"172.21.15.27:6819/3166119382","cluster_addr":"172.21.15.27:6821/3166119382","heartbeat_back_addr":"172.21.15.27:6825/3166119382","heartbeat_front_addr":"172.21.15.27:6823/3166119382","state":["exists","up"]},{"osd":3,"uuid":"9e66dcd4-23e5-42b7-b665-8fa128130229","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6827","nonce":1975358975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6829","nonce":1975358975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6832","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6833","nonce":1975358975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":1975358975},{"type":"v1","addr":"172.21.15.27:6831","nonce":1975358975}]},"public_addr":"172.21.15.27:6827/1975358975","cluster_addr":"172.21.15.27:6829/1975358975","heartbeat_back_addr":"172.21.15.27:6833/1975358975","heartbeat_front_addr":"172.21.15.27:6831/1975358975","state":["exists","up"]},{"osd":4,"uuid":"34b2e281-2987-41a2-adc6-fb895ded7659","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6801","nonce":4256538191}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6803","nonce":4256538191}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6807","nonce":4256538191}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":4256538191},{"type":"v1","addr":"172.21.15.105:6805","nonce":4256538191}]},"public_addr":"172.21.15.105:6801/4256538191","cluster_addr":"172.21.15.105:6803/4256538191","heartbeat_back_addr":"172.21.15.105:6807/4256538191","heartbeat_front_addr":"172.21.15.105:6805/4256538191","state":["exists","up"]},{"osd":5,"uuid":"73eb3b08-0c61-45b2-a7c2-ceac17ef7982","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6809","nonce":1140342962}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6811","nonce":1140342962}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6815","nonce":1140342962}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1140342962},{"type":"v1","addr":"172.21.15.105:6813","nonce":1140342962}]},"public_addr":"172.21.15.105:6809/1140342962","cluster_addr":"172.21.15.105:6811/1140342962","heartbeat_back_addr":"172.21.15.105:6815/1140342962","heartbeat_front_addr":"172.21.15.105:6813/1140342962","state":["exists","up"]},{"osd":6,"uuid":"ecf7dae8-bb99-4250-8e53-87d2929b6799","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6817","nonce":2946684313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6819","nonce":2946684313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6823","nonce":2946684313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":2946684313},{"type":"v1","addr":"172.21.15.105:6821","nonce":2946684313}]},"public_addr":"172.21.15.105:6817/2946684313","cluster_addr":"172.21.15.105:6819/2946684313","heartbeat_back_addr":"172.21.15.105:6823/2946684313","heartbeat_front_addr":"172.21.15.105:6821/2946684313","state":["exists","up"]},{"osd":7,"uuid":"e7bce8ad-ddc0-4d0c-ab1a-4d010f63b169","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6825","nonce":3306013433}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6827","nonce":3306013433}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6831","nonce":3306013433}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3306013433},{"type":"v1","addr":"172.21.15.105:6829","nonce":3306013433}]},"public_addr":"172.21.15.105:6825/3306013433","cluster_addr":"172.21.15.105:6827/3306013433","heartbeat_back_addr":"172.21.15.105:6831/3306013433","heartbeat_front_addr":"172.21.15.105:6829/3306013433","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:17.322071+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:35.898970+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:45:55.114365+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:15.160089+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:30.830357+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:46:48.633705+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:06.919383+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138297136906239,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T08:47:25.372879+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/466285564":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6800/3971890029":"2024-09-18T08:44:10.699554+0000","172.21.15.27:0/1001596001":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/277636790":"2024-09-18T08:43:51.105114+0000","172.21.15.27:6800/3868462814":"2024-09-18T08:43:51.105114+0000","172.21.15.27:0/3348792656":"2024-09-18T08:44:10.699554+0000","172.21.15.27:6801/3868462814":"2024-09-18T08:43:51.105114+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T08:48:00.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:47:58.790902+0000 mgr.y (mgr.14144) 169 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:47:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:47:59.438850+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.27:0/1984492256' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:48:00.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:47:58.790902+0000 mgr.y (mgr.14144) 169 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:00.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:47:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:47:59.438850+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.27:0/1984492256' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:48:00.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T08:48:00.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T08:48:00.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T08:48:00.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T08:48:00.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T08:48:00.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T08:48:00.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T08:48:00.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T08:48:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:47:58.790902+0000 mgr.y (mgr.14144) 169 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:47:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:47:59.438850+0000 mon.c (mon.1) 14 : audit [DBG] from='client.? 172.21.15.27:0/1984492256' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T08:48:00.461 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.462 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.463 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.464 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.471 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.472 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.476 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:00.489 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:02.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:00.791215+0000 mgr.y (mgr.14144) 170 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:02.138 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:00.791215+0000 mgr.y (mgr.14144) 170 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:02.139 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:00.791215+0000 mgr.y (mgr.14144) 170 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:03.892 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:02.791572+0000 mgr.y (mgr.14144) 171 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:03.893 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:02.791572+0000 mgr.y (mgr.14144) 171 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:02.791572+0000 mgr.y (mgr.14144) 171 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:04.508 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:04 smithi027 bash[38545]: Copying blob sha256:3589a6efd9ce96b2f2218a80bacb84f60d5e584d024023617b0f56b87fe798c3 2024-09-17T08:48:06.214 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:04.791910+0000 mgr.y (mgr.14144) 172 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:06.214 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:04.791910+0000 mgr.y (mgr.14144) 172 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:06.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:04.791910+0000 mgr.y (mgr.14144) 172 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:06.507 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:06 smithi027 bash[38545]: Copying config sha256:e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 2024-09-17T08:48:07.007 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:06 smithi027 bash[38545]: Writing manifest to image destination 2024-09-17T08:48:07.351 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:07 smithi027 podman[38545]: 2024-09-17 08:48:07.123103072 +0000 UTC m=+9.371127621 image pull e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 docker.io/prom/node-exporter:v0.18.1 2024-09-17T08:48:07.945 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:06.792208+0000 mgr.y (mgr.14144) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:07.945 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:06.792208+0000 mgr.y (mgr.14144) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:06.792208+0000 mgr.y (mgr.14144) 173 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:08.792648+0000 mgr.y (mgr.14144) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:08.792648+0000 mgr.y (mgr.14144) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:10.258 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:09 smithi027 podman[38545]: 2024-09-17 08:48:09.901090032 +0000 UTC m=+12.149114606 container create e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:08.792648+0000 mgr.y (mgr.14144) 174 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:11.724 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 podman[38545]: 2024-09-17 08:48:11.529993359 +0000 UTC m=+13.778017908 container init e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:11.724 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 podman[38545]: 2024-09-17 08:48:11.533661506 +0000 UTC m=+13.781686041 container start e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:11.724 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - arp" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - bcache" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - bonding" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - cpu" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - edac" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - entropy" source="node_exporter.go:104" 2024-09-17T08:48:11.725 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - filefd" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - netclass" source="node_exporter.go:104" 2024-09-17T08:48:11.726 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - netdev" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - netstat" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - nfs" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - pressure" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - stat" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - textfile" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - time" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - uname" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - xfs" source="node_exporter.go:104" 2024-09-17T08:48:11.727 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg=" - zfs" source="node_exporter.go:104" 2024-09-17T08:48:11.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[39438]: time="2024-09-17T08:48:11Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2024-09-17T08:48:11.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 bash[38545]: e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 2024-09-17T08:48:11.728 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:48:11 smithi027 systemd[1]: Started Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:48:12.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:10.792911+0000 mgr.y (mgr.14144) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:10.884270+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:10.890746+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:11.684819+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:10.792911+0000 mgr.y (mgr.14144) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:10.884270+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:10.890746+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:48:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:11.684819+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:12.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:10.792911+0000 mgr.y (mgr.14144) 175 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:12.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:10.884270+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:48:12.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:10.890746+0000 mon.a (mon.0) 465 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:48:12.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:11.684819+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:12.525 INFO:teuthology.orchestra.run.smithi027.stdout:180388626446 2024-09-17T08:48:12.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T08:48:12.527 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:12 smithi105 bash[36056]: Trying to pull docker.io/prom/node-exporter:v0.18.1... 2024-09-17T08:48:13.004 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:48:11.685488+0000 mgr.y (mgr.14144) 176 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T08:48:13.005 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:48:11.685488+0000 mgr.y (mgr.14144) 176 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T08:48:13.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:48:11.685488+0000 mgr.y (mgr.14144) 176 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T08:48:14.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:12.793187+0000 mgr.y (mgr.14144) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:14.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:12.793187+0000 mgr.y (mgr.14144) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:14.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:12.793187+0000 mgr.y (mgr.14144) 177 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:15.669 INFO:teuthology.orchestra.run.smithi027.stdout:107374182427 2024-09-17T08:48:15.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T08:48:16.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:14.793672+0000 mgr.y (mgr.14144) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:16.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:14.793672+0000 mgr.y (mgr.14144) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:16.048 INFO:teuthology.orchestra.run.smithi027.stdout:77309411357 2024-09-17T08:48:16.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T08:48:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:14.793672+0000 mgr.y (mgr.14144) 178 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:16.453 INFO:teuthology.orchestra.run.smithi027.stdout:128849018902 2024-09-17T08:48:16.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T08:48:17.680 INFO:teuthology.orchestra.run.smithi027.stdout:55834574883 2024-09-17T08:48:17.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T08:48:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:16.793974+0000 mgr.y (mgr.14144) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:16.793974+0000 mgr.y (mgr.14144) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:18.086 INFO:teuthology.orchestra.run.smithi027.stdout:34359738407 2024-09-17T08:48:18.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T08:48:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:16.793974+0000 mgr.y (mgr.14144) 179 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:18.497 INFO:teuthology.orchestra.run.smithi027.stdout:154618822675 2024-09-17T08:48:18.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T08:48:18.845 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.845 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.846 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.846 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.847 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.852 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:18.884 INFO:teuthology.orchestra.run.smithi027.stdout:206158430219 2024-09-17T08:48:18.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T08:48:18.910 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:19.277 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:18 smithi105 bash[36056]: Getting image source signatures 2024-09-17T08:48:19.277 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:18 smithi105 bash[36056]: Copying blob sha256:190160031744b0810999dd6f293478be0b41524654c70782cf09437095430f2f 2024-09-17T08:48:19.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:18 smithi105 bash[36056]: Copying blob sha256:49a2d53aa1af115e87f7e23a7fe4b7a3e12304d37982fb6528b035d0f68a3b56 2024-09-17T08:48:19.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:18 smithi105 bash[36056]: Copying blob sha256:3589a6efd9ce96b2f2218a80bacb84f60d5e584d024023617b0f56b87fe798c3 2024-09-17T08:48:19.986 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:18.794227+0000 mgr.y (mgr.14144) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:19.987 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:18.794227+0000 mgr.y (mgr.14144) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:20.269 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:18.794227+0000 mgr.y (mgr.14144) 180 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:20.528 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 bash[36056]: Copying config sha256:e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 2024-09-17T08:48:20.528 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 bash[36056]: Writing manifest to image destination 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 podman[36056]: 2024-09-17 08:48:20.740322183 +0000 UTC m=+8.429635443 container create 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 podman[36056]: 2024-09-17 08:48:20.667164327 +0000 UTC m=+8.356477594 image pull e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 docker.io/prom/node-exporter:v0.18.1 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 podman[36056]: 2024-09-17 08:48:20.983682964 +0000 UTC m=+8.672996229 container init 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 podman[36056]: 2024-09-17 08:48:20.986950113 +0000 UTC m=+8.676263368 container start 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - arp" source="node_exporter.go:104" 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - bcache" source="node_exporter.go:104" 2024-09-17T08:48:20.991 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - bonding" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - cpu" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - edac" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - entropy" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - filefd" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2024-09-17T08:48:20.992 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - netclass" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - netdev" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - netstat" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - nfs" source="node_exporter.go:104" 2024-09-17T08:48:21.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - pressure" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - stat" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - textfile" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - time" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - uname" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - xfs" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg=" - zfs" source="node_exporter.go:104" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[36132]: time="2024-09-17T08:48:20Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:21 smithi105 bash[36056]: 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 2024-09-17T08:48:21.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:48:21 smithi105 systemd[1]: Started Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:48:22.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:20.794547+0000 mgr.y (mgr.14144) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:22.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:21.154762+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:22.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:48:21.160593+0000 mgr.y (mgr.14144) 182 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T08:48:22.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:20.794547+0000 mgr.y (mgr.14144) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:22.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:21.154762+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:22.370 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:48:21.160593+0000 mgr.y (mgr.14144) 182 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T08:48:22.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:20.794547+0000 mgr.y (mgr.14144) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:22.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:21.154762+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:22.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:48:21.160593+0000 mgr.y (mgr.14144) 182 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T08:48:23.293 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:23.465 INFO:teuthology.orchestra.run.smithi027.stdout:180388626449 2024-09-17T08:48:23.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:22.794950+0000 mgr.y (mgr.14144) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:23.984 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:23.465264+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.27:0/2193359575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T08:48:23.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:22.794950+0000 mgr.y (mgr.14144) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:23.984 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:23.465264+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.27:0/2193359575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T08:48:24.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:22.794950+0000 mgr.y (mgr.14144) 183 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:23.465264+0000 mon.a (mon.0) 468 : audit [DBG] from='client.? 172.21.15.27:0/2193359575' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T08:48:24.379 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626446 got 180388626449 for osd.6 2024-09-17T08:48:24.379 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:25.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:24.795374+0000 mgr.y (mgr.14144) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:25.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:24.795374+0000 mgr.y (mgr.14144) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:26.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:24.795374+0000 mgr.y (mgr.14144) 184 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:26.479 INFO:teuthology.orchestra.run.smithi027.stdout:34359738408 2024-09-17T08:48:27.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:26.479315+0000 mon.a (mon.0) 469 : audit [DBG] from='client.? 172.21.15.27:0/177688269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T08:48:27.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:26.479315+0000 mon.a (mon.0) 469 : audit [DBG] from='client.? 172.21.15.27:0/177688269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T08:48:27.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:26.479315+0000 mon.a (mon.0) 469 : audit [DBG] from='client.? 172.21.15.27:0/177688269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T08:48:27.661 INFO:teuthology.orchestra.run.smithi027.stdout:55834574885 2024-09-17T08:48:27.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:26.795615+0000 mgr.y (mgr.14144) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:27.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:27.661096+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.27:0/445243578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T08:48:27.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:26.795615+0000 mgr.y (mgr.14144) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:27.977 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:27.661096+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.27:0/445243578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T08:48:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:26.795615+0000 mgr.y (mgr.14144) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:27.661096+0000 mon.c (mon.1) 15 : audit [DBG] from='client.? 172.21.15.27:0/445243578' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T08:48:28.333 INFO:teuthology.orchestra.run.smithi027.stdout:77309411360 2024-09-17T08:48:28.912 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:28.333135+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.27:0/2487237468' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T08:48:28.912 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:28.333135+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.27:0/2487237468' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T08:48:28.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411357 got 77309411360 for osd.2 2024-09-17T08:48:28.994 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:28.333135+0000 mon.a (mon.0) 470 : audit [DBG] from='client.? 172.21.15.27:0/2487237468' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T08:48:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:28.795912+0000 mgr.y (mgr.14144) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:28.795912+0000 mgr.y (mgr.14144) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:30.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:28.795912+0000 mgr.y (mgr.14144) 186 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:30.884 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574883 got 55834574885 for osd.1 2024-09-17T08:48:30.884 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:31.765 INFO:teuthology.orchestra.run.smithi027.stdout:206158430223 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:30.796248+0000 mgr.y (mgr.14144) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:31.676177+0000 mon.a (mon.0) 471 : audit [DBG] from='client.? 172.21.15.27:0/773103052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:31.764733+0000 mon.a (mon.0) 472 : audit [DBG] from='client.? 172.21.15.27:0/3199321539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:30.796248+0000 mgr.y (mgr.14144) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:31.676177+0000 mon.a (mon.0) 471 : audit [DBG] from='client.? 172.21.15.27:0/773103052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T08:48:31.999 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:31.764733+0000 mon.a (mon.0) 472 : audit [DBG] from='client.? 172.21.15.27:0/3199321539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T08:48:32.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:30.796248+0000 mgr.y (mgr.14144) 187 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:32.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:31.676177+0000 mon.a (mon.0) 471 : audit [DBG] from='client.? 172.21.15.27:0/773103052' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T08:48:32.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:31.764733+0000 mon.a (mon.0) 472 : audit [DBG] from='client.? 172.21.15.27:0/3199321539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T08:48:32.244 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738407 got 34359738408 for osd.0 2024-09-17T08:48:32.244 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:32.582 INFO:teuthology.orchestra.run.smithi027.stdout:128849018905 2024-09-17T08:48:33.007 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430219 got 206158430223 for osd.7 2024-09-17T08:48:33.007 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:33.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:32.581954+0000 mon.a (mon.0) 473 : audit [DBG] from='client.? 172.21.15.27:0/2811288327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T08:48:33.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:32.581954+0000 mon.a (mon.0) 473 : audit [DBG] from='client.? 172.21.15.27:0/2811288327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T08:48:33.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:32.581954+0000 mon.a (mon.0) 473 : audit [DBG] from='client.? 172.21.15.27:0/2811288327' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T08:48:33.335 INFO:teuthology.orchestra.run.smithi027.stdout:154618822679 2024-09-17T08:48:33.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:32.796540+0000 mgr.y (mgr.14144) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:33.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:32.796540+0000 mgr.y (mgr.14144) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:33.997 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822675 got 154618822679 for osd.5 2024-09-17T08:48:33.997 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:32.796540+0000 mgr.y (mgr.14144) 188 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:34.602 INFO:teuthology.orchestra.run.smithi027.stdout:107374182431 2024-09-17T08:48:34.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018902 got 128849018905 for osd.4 2024-09-17T08:48:34.861 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:35.098 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:34.602710+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.27:0/4010624365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T08:48:35.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:34.602710+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.27:0/4010624365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T08:48:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:34.602710+0000 mon.c (mon.1) 16 : audit [DBG] from='client.? 172.21.15.27:0/4010624365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T08:48:35.419 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182427 got 107374182431 for osd.3 2024-09-17T08:48:35.420 DEBUG:teuthology.parallel:result is None 2024-09-17T08:48:35.420 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T08:48:35.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T08:48:35.634 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:36.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:34.796840+0000 mgr.y (mgr.14144) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:34.796840+0000 mgr.y (mgr.14144) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:34.796840+0000 mgr.y (mgr.14144) 189 : cluster [DBG] pgmap v172: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:37.402 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:48:37.403 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-09-17T08:48:37.987 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":173,"stamp":"2024-09-17T08:48:36.796992+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43220,"kb_used_data":3732,"kb_used_omap":0,"kb_used_meta":39424,"kb_avail":749917996,"statfs":{"total":767960285184,"available":767916027904,"internally_reserved":0,"allocated":3821568,"data_stored":1511015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40370176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":"10","reported_epoch":"50","state":"active+clean","last_fresh":"2024-09-17T08:47:29.401787+0000","last_change":"2024-09-17T08:47:29.401787+0000","last_active":"2024-09-17T08:47:29.401787+0000","last_peered":"2024-09-17T08:47:29.401787+0000","last_clean":"2024-09-17T08:47:29.401787+0000","last_became_active":"2024-09-17T08:47:29.401466+0000","last_became_peered":"2024-09-17T08:47:29.401466+0000","last_unstale":"2024-09-17T08:47:29.401787+0000","last_undegraded":"2024-09-17T08:47:29.401787+0000","last_fullsized":"2024-09-17T08:47:29.401787+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T08:45:58.092490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T08:45:58.092490+0000","last_clean_scrub_stamp":"2024-09-17T08:45:58.092490+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430224,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5212,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739940,"statfs":{"total":95995035648,"available":95989698560,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.376},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.53500000000000003}]},{"osd":1,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.55900000000000005}]},{"osd":2,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.49199999999999999}]},{"osd":3,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.66500000000000004}]},{"osd":4,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.61399999999999999}]},{"osd":5,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.628},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.60299999999999998}]},{"osd":6,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.65300000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626452,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5276,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.47999999999999998}]},{"osd":1,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.68600000000000005}]},{"osd":2,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.63700000000000001}]},{"osd":3,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.61699999999999999}]},{"osd":4,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.60399999999999998}]},{"osd":5,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.65100000000000002}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.63},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.65900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574887,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5524,"kb_used_data":460,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739628,"statfs":{"total":95995035648,"available":95989379072,"internally_reserved":0,"allocated":471040,"data_stored":188365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35399999999999998,"15min":0.34399999999999997},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.48399999999999999}]},{"osd":2,"last update":"Tue Sep 17 08:47:58 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42399999999999999,"15min":0.42199999999999999},"min":{"1min":0.29799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59899999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.41899999999999998},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.57099999999999995}]},{"osd":3,"last update":"Tue Sep 17 08:48:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57099999999999995,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.442,"5min":0.47999999999999998,"15min":0.48599999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.66900000000000004,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55300000000000005}]},{"osd":4,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.78600000000000003}]},{"osd":5,"last update":"Tue Sep 17 08:47:53 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.82299999999999995}]},{"osd":6,"last update":"Tue Sep 17 08:48:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.64000000000000001}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.74399999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5596,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739556,"statfs":{"total":95995035648,"available":95989305344,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 08:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.38,"15min":0.375},"min":{"1min":0.25,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.47199999999999998}]},{"osd":2,"last update":"Tue Sep 17 08:48:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.40500000000000003,"15min":0.40600000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.52800000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.45800000000000002}]},{"osd":3,"last update":"Tue Sep 17 08:48:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.55700000000000005,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46200000000000002,"15min":0.46700000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54300000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.52200000000000002}]},{"osd":4,"last update":"Tue Sep 17 08:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54300000000000004,"15min":0.52900000000000003},"min":{"1min":0.44400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.59599999999999997,"15min":0.58999999999999997},"min":{"1min":0.438,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.752}]},{"osd":5,"last update":"Tue Sep 17 08:47:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.69899999999999995}]},{"osd":6,"last update":"Tue Sep 17 08:48:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.70899999999999996}]},{"osd":7,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.79300000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411362,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5468,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739684,"statfs":{"total":95995035648,"available":95989436416,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.38100000000000001},"min":{"1min":0.26100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.47899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.311},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.58599999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.37}]},{"osd":1,"last update":"Tue Sep 17 08:48:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.40000000000000002,"15min":0.40100000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.255,"5min":0.223,"15min":0.223},"max":{"1min":0.55900000000000005,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.39300000000000002}]},{"osd":3,"last update":"Tue Sep 17 08:48:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.45500000000000002,"15min":0.46400000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59099999999999997,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.47199999999999998,"15min":0.48299999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.61899999999999999,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40600000000000003}]},{"osd":4,"last update":"Tue Sep 17 08:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.60899999999999999}]},{"osd":5,"last update":"Tue Sep 17 08:47:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":8.2159999999999993,"5min":8.2159999999999993,"15min":8.2159999999999993},"last":0.64700000000000002}]},{"osd":6,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.61799999999999999}]},{"osd":7,"last update":"Tue Sep 17 08:48:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.65400000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182431,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55700000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45100000000000001,"15min":0.45600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.33100000000000002}]},{"osd":1,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42599999999999999,"15min":0.42699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.37},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.435,"15min":0.442},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56499999999999995,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.40300000000000002}]},{"osd":2,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.44},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.47299999999999998,"15min":0.47699999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.34499999999999997}]},{"osd":4,"last update":"Tue Sep 17 08:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.55400000000000005}]},{"osd":5,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.70499999999999996}]},{"osd":6,"last update":"Tue Sep 17 08:48:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.51800000000000002}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.63600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018906,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.496}]},{"osd":1,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.64500000000000002}]},{"osd":2,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.52000000000000002}]},{"osd":3,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.66200000000000003}]},{"osd":5,"last update":"Tue Sep 17 08:47:51 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.67700000000000005}]},{"osd":6,"last update":"Tue Sep 17 08:48:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.79800000000000004}]},{"osd":7,"last update":"Tue Sep 17 08:48:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.75}]}]},{"osd":5,"up_from":36,"seq":154618822680,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5336,"kb_used_data":464,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":475136,"data_stored":188680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.581,"5min":3.581,"15min":3.581},"last":0.627},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.69799999999999995}]},{"osd":1,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.66200000000000003}]},{"osd":2,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":3.6349999999999998,"5min":3.6349999999999998,"15min":3.6349999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.70899999999999996}]},{"osd":3,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.67300000000000004}]},{"osd":4,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53800000000000003}]},{"osd":6,"last update":"Tue Sep 17 08:48:08 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.64500000000000002}]},{"osd":7,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.73199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T08:48:37.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T08:48:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:36.797140+0000 mgr.y (mgr.14144) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:37.399127+0000 mgr.y (mgr.14144) 191 : audit [DBG] from='client.24487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:36.797140+0000 mgr.y (mgr.14144) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:37.399127+0000 mgr.y (mgr.14144) 191 : audit [DBG] from='client.24487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:38.193 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:36.797140+0000 mgr.y (mgr.14144) 190 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:37.399127+0000 mgr.y (mgr.14144) 191 : audit [DBG] from='client.24487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:40.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:38.797445+0000 mgr.y (mgr.14144) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:38.797445+0000 mgr.y (mgr.14144) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:40.016 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:48:40.017 INFO:teuthology.orchestra.run.smithi027.stderr:dumped all 2024-09-17T08:48:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:38.797445+0000 mgr.y (mgr.14144) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:41.006 INFO:teuthology.orchestra.run.smithi027.stdout:{"pg_ready":true,"pg_map":{"version":174,"stamp":"2024-09-17T08:48:38.797282+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":43220,"kb_used_data":3732,"kb_used_omap":0,"kb_used_meta":39424,"kb_avail":749917996,"statfs":{"total":767960285184,"available":767916027904,"internally_reserved":0,"allocated":3821568,"data_stored":1511015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40370176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"1.0","version":"0'0","reported_seq":"10","reported_epoch":"50","state":"active+clean","last_fresh":"2024-09-17T08:47:29.401787+0000","last_change":"2024-09-17T08:47:29.401787+0000","last_active":"2024-09-17T08:47:29.401787+0000","last_peered":"2024-09-17T08:47:29.401787+0000","last_clean":"2024-09-17T08:47:29.401787+0000","last_became_active":"2024-09-17T08:47:29.401466+0000","last_became_peered":"2024-09-17T08:47:29.401466+0000","last_unstale":"2024-09-17T08:47:29.401787+0000","last_undegraded":"2024-09-17T08:47:29.401787+0000","last_fullsized":"2024-09-17T08:47:29.401787+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T08:45:58.092490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T08:45:58.092490+0000","last_clean_scrub_stamp":"2024-09-17T08:45:58.092490+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430225,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5212,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4736,"kb_avail":93739940,"statfs":{"total":95995035648,"available":95989698560,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4849664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51400000000000001}]},{"osd":1,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.48699999999999999}]},{"osd":2,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.622}]},{"osd":3,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.60499999999999998}]},{"osd":4,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.72499999999999998}]},{"osd":5,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.64600000000000002}]},{"osd":6,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.75700000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626453,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5276,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4800,"kb_avail":93739876,"statfs":{"total":95995035648,"available":95989633024,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4915200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52800000000000002}]},{"osd":1,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.55500000000000005}]},{"osd":2,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.55000000000000004}]},{"osd":3,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.52200000000000002}]},{"osd":4,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"last":0.5},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.437}]},{"osd":5,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.53800000000000003}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.60699999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574887,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5524,"kb_used_data":460,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739628,"statfs":{"total":95995035648,"available":95989379072,"internally_reserved":0,"allocated":471040,"data_stored":188365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.35399999999999998,"15min":0.34399999999999997},"min":{"1min":0.26900000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28999999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.48399999999999999}]},{"osd":2,"last update":"Tue Sep 17 08:47:58 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42399999999999999,"15min":0.42199999999999999},"min":{"1min":0.29799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59899999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.42299999999999999,"15min":0.41899999999999998},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.57099999999999995}]},{"osd":3,"last update":"Tue Sep 17 08:48:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57099999999999995,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.442,"5min":0.47999999999999998,"15min":0.48599999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.66900000000000004,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.55300000000000005}]},{"osd":4,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.78600000000000003}]},{"osd":5,"last update":"Tue Sep 17 08:47:53 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.82299999999999995}]},{"osd":6,"last update":"Tue Sep 17 08:48:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.64000000000000001}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.74399999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5596,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739556,"statfs":{"total":95995035648,"available":95989305344,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 08:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.371,"15min":0.36499999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.38,"15min":0.375},"min":{"1min":0.25,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.47199999999999998}]},{"osd":2,"last update":"Tue Sep 17 08:48:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.40500000000000003,"15min":0.40600000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.52800000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.42199999999999999,"15min":0.41599999999999998},"min":{"1min":0.26900000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.45800000000000002}]},{"osd":3,"last update":"Tue Sep 17 08:48:19 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.55700000000000005,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46200000000000002,"15min":0.46700000000000003},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54300000000000004,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.52200000000000002}]},{"osd":4,"last update":"Tue Sep 17 08:48:34 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54300000000000004,"15min":0.52900000000000003},"min":{"1min":0.44400000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.629,"5min":0.59599999999999997,"15min":0.58999999999999997},"min":{"1min":0.438,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.752}]},{"osd":5,"last update":"Tue Sep 17 08:47:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.69899999999999995}]},{"osd":6,"last update":"Tue Sep 17 08:48:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.70899999999999996}]},{"osd":7,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.79300000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411363,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5468,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739684,"statfs":{"total":95995035648,"available":95989436416,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.379,"15min":0.38100000000000001},"min":{"1min":0.26100000000000001,"5min":0.248,"15min":0.248},"max":{"1min":0.47899999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.40699999999999997,"15min":0.40899999999999997},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.58599999999999997,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.33600000000000002}]},{"osd":1,"last update":"Tue Sep 17 08:48:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.40000000000000002,"15min":0.40100000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.255,"5min":0.223,"15min":0.223},"max":{"1min":0.55900000000000005,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.42999999999999999}]},{"osd":3,"last update":"Tue Sep 17 08:48:20 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.45500000000000002,"15min":0.46400000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.59099999999999997,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.37},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.47199999999999998,"15min":0.48299999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.61899999999999999,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.40100000000000002}]},{"osd":4,"last update":"Tue Sep 17 08:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.57799999999999996,"15min":0.57399999999999995},"min":{"1min":0.47099999999999997,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.42099999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.76900000000000002,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.78600000000000003}]},{"osd":5,"last update":"Tue Sep 17 08:47:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":8.2159999999999993,"5min":8.2159999999999993,"15min":8.2159999999999993},"last":0.748}]},{"osd":6,"last update":"Tue Sep 17 08:48:13 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.64800000000000002}]},{"osd":7,"last update":"Tue Sep 17 08:48:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.72999999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55700000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45100000000000001,"15min":0.45600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65000000000000002,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.35699999999999998}]},{"osd":1,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.42599999999999999,"15min":0.42699999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.435,"15min":0.442},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56499999999999995,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.32400000000000001}]},{"osd":2,"last update":"Tue Sep 17 08:48:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.49199999999999999,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.371},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.47299999999999998,"15min":0.47699999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.33600000000000002}]},{"osd":4,"last update":"Tue Sep 17 08:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.63100000000000001,"15min":0.63700000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.026,"5min":1.026,"15min":1.026},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61599999999999999,"15min":0.61799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.82099999999999995,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63}]},{"osd":5,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Tue Sep 17 08:48:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.55200000000000005}]},{"osd":7,"last update":"Tue Sep 17 08:48:29 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.629,"5min":0.629,"15min":0.629},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.52800000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5404,"kb_used_data":468,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739748,"statfs":{"total":95995035648,"available":95989501952,"internally_reserved":0,"allocated":479232,"data_stored":188995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69499999999999995}]},{"osd":1,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.71599999999999997}]},{"osd":2,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.60399999999999998}]},{"osd":3,"last update":"Tue Sep 17 08:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.69899999999999995}]},{"osd":5,"last update":"Tue Sep 17 08:47:51 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.68200000000000005}]},{"osd":6,"last update":"Tue Sep 17 08:48:11 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.56899999999999995}]},{"osd":7,"last update":"Tue Sep 17 08:48:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.58299999999999996}]}]},{"osd":5,"up_from":36,"seq":154618822680,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5336,"kb_used_data":464,"kb_used_omap":0,"kb_used_meta":4864,"kb_avail":93739816,"statfs":{"total":95995035648,"available":95989571584,"internally_reserved":0,"allocated":475136,"data_stored":188680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":4980736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.581,"5min":3.581,"15min":3.581},"last":0.627},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.69799999999999995}]},{"osd":1,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.66200000000000003}]},{"osd":2,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":3.6349999999999998,"5min":3.6349999999999998,"15min":3.6349999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.70899999999999996}]},{"osd":3,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.67300000000000004}]},{"osd":4,"last update":"Tue Sep 17 08:47:50 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53800000000000003}]},{"osd":6,"last update":"Tue Sep 17 08:48:08 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.64500000000000002}]},{"osd":7,"last update":"Tue Sep 17 08:48:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.73199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T08:48:41.008 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T08:48:41.008 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T08:48:41.008 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T08:48:41.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T08:48:41.254 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/mon.a/config 2024-09-17T08:48:41.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:40.013065+0000 mgr.y (mgr.14144) 193 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:41.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:40.013065+0000 mgr.y (mgr.14144) 193 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:41.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:40.013065+0000 mgr.y (mgr.14144) 193 : audit [DBG] from='client.14559 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T08:48:42.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:40.797788+0000 mgr.y (mgr.14144) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:40.797788+0000 mgr.y (mgr.14144) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:40.797788+0000 mgr.y (mgr.14144) 194 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:43.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:42.798189+0000 mgr.y (mgr.14144) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:43.836 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:42.798189+0000 mgr.y (mgr.14144) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:43.836 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:48:44.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:42.798189+0000 mgr.y (mgr.14144) 195 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:44.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:43.834550+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.27:0/3275346588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T08:48:44.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:43.834550+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.27:0/3275346588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T08:48:45.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:43.834550+0000 mon.c (mon.1) 17 : audit [DBG] from='client.? 172.21.15.27:0/3275346588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T08:48:45.129 INFO:teuthology.orchestra.run.smithi027.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T08:48:45.130 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T08:48:45.130 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T08:48:45.130 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T08:48:45.140 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-09-17T08:48:45.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-09-17T08:48:45.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:44.798680+0000 mgr.y (mgr.14144) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:45.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:44.798680+0000 mgr.y (mgr.14144) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:46.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:44.798680+0000 mgr.y (mgr.14144) 196 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:48.075 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:46.799074+0000 mgr.y (mgr.14144) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:48.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:46.799074+0000 mgr.y (mgr.14144) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:46.799074+0000 mgr.y (mgr.14144) 197 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:49.257 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:48 smithi027 systemd[1]: Starting Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:48:49.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:48.799509+0000 mgr.y (mgr.14144) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:49.976 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:48.799509+0000 mgr.y (mgr.14144) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:49.976 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:49 smithi027 podman[41737]: 2024-09-17 08:48:49.540309775 +0000 UTC m=+0.018115856 image pull 0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0 docker.io/prom/alertmanager:v0.20.0 2024-09-17T08:48:49.976 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:49 smithi027 podman[41737]: 2024-09-17 08:48:49.659628556 +0000 UTC m=+0.137434629 volume create 2074cb979df578cb7fcc985709b6ea2eac9e2a8cc6099616bdd1e98cc70beff8 2024-09-17T08:48:49.977 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:49 smithi027 podman[41737]: 2024-09-17 08:48:49.76005762 +0000 UTC m=+0.237863689 container create 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:48.799509+0000 mgr.y (mgr.14144) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 podman[41737]: 2024-09-17 08:48:50.255650184 +0000 UTC m=+0.733456254 container init 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 podman[41737]: 2024-09-17 08:48:50.258863361 +0000 UTC m=+0.736669437 container start 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.279Z caller=main.go:231 msg="Starting Alertmanager" version="(version=0.20.0, branch=HEAD, revision=f74be0400a6243d10bb53812d6fa408ad71ff32d)" 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.279Z caller=main.go:232 build_context="(go=go1.13.5, user=root@00c3106655f8, date=20191211-14:13:14)" 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.280Z caller=cluster.go:161 component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.281Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.308Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:48:50.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.308Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:48:50.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:50.311Z caller=main.go:497 msg=Listening address=:9093 2024-09-17T08:48:50.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 bash[41737]: 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 2024-09-17T08:48:50.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:50 smithi027 systemd[1]: Started Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:48:51.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.557474+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.639274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.897 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.644864+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.645286+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.651495+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:50.868349+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.869738+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.27:0/1820771961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.871743+0000 mon.a (mon.0) 480 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.557474+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.639274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.644864+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.645286+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.651495+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:50.868349+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.869738+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.27:0/1820771961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:51.899 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.871743+0000 mon.a (mon.0) 480 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "acd7501c-bb75-4ce7-b40c-cb4e10f79740", 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "r", 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout: "current_period": "3e017442-65e2-4dc5-8a63-dcb415d51154", 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout: "epoch": 1 2024-09-17T08:48:51.900 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.557474+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.639274+0000 mon.a (mon.0) 475 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.644864+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.645286+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.651495+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:50.868349+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.869738+0000 mon.c (mon.1) 18 : audit [INF] from='client.? 172.21.15.27:0/1820771961' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.871743+0000 mon.a (mon.0) 480 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-09-17T08:48:52.507 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:48:52.281Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000071152s 2024-09-17T08:48:52.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-09-17T08:48:52.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:50.645854+0000 mgr.y (mgr.14144) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:52.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:48:50.654910+0000 mgr.y (mgr.14144) 200 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T08:48:52.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:50.800057+0000 mgr.y (mgr.14144) 201 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:52.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:48:51.865236+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T08:48:52.902 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:51.865379+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-17T08:48:52.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:50.645854+0000 mgr.y (mgr.14144) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:52.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:48:50.654910+0000 mgr.y (mgr.14144) 200 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T08:48:52.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:50.800057+0000 mgr.y (mgr.14144) 201 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:52.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:48:51.865236+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T08:48:52.903 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:51.865379+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-17T08:48:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:50.645854+0000 mgr.y (mgr.14144) 199 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:48:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:48:50.654910+0000 mgr.y (mgr.14144) 200 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T08:48:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:50.800057+0000 mgr.y (mgr.14144) 201 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:48:51.865236+0000 mon.a (mon.0) 481 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-09-17T08:48:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:51.865379+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-17T08:48:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:52.800363+0000 mgr.y (mgr.14144) 202 : cluster [DBG] pgmap v183: 33 pgs: 28 unknown, 5 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:52.865793+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-17T08:48:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:52.800363+0000 mgr.y (mgr.14144) 202 : cluster [DBG] pgmap v183: 33 pgs: 28 unknown, 5 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:52.865793+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-17T08:48:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:52.800363+0000 mgr.y (mgr.14144) 202 : cluster [DBG] pgmap v183: 33 pgs: 28 unknown, 5 active+clean; 0 B data, 42 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:52.865793+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "32cf48b3-07cf-42cc-a6a7-af55b3fac290", 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default", 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout: "api_name": "default", 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout: "is_master": "true", 2024-09-17T08:48:54.671 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames": [], 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames_s3website": [], 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "", 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "zones": [], 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_targets": [], 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "default_placement": "", 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "acd7501c-bb75-4ce7-b40c-cb4e10f79740", 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_policy": { 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: "groups": [] 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:48:54.672 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:48:55.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-09-17T08:48:56.120 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:54.800702+0000 mgr.y (mgr.14144) 203 : cluster [DBG] pgmap v185: 33 pgs: 7 unknown, 26 active+clean; 400 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:56.120 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:54.800702+0000 mgr.y (mgr.14144) 203 : cluster [DBG] pgmap v185: 33 pgs: 7 unknown, 26 active+clean; 400 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:54.800702+0000 mgr.y (mgr.14144) 203 : cluster [DBG] pgmap v185: 33 pgs: 7 unknown, 26 active+clean; 400 B data, 43 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:48:57.135 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:48:57.135 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "5c0596db-6a9c-4cdf-bf18-538d78a96a23", 2024-09-17T08:48:57.135 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "z", 2024-09-17T08:48:57.135 INFO:teuthology.orchestra.run.smithi027.stdout: "domain_root": "z.rgw.meta:root", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "control_pool": "z.rgw.control", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "gc_pool": "z.rgw.log:gc", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "lc_pool": "z.rgw.log:lc", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "log_pool": "z.rgw.log", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "roles_pool": "z.rgw.meta:roles", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-09-17T08:48:57.136 INFO:teuthology.orchestra.run.smithi027.stdout: "otp_pool": "z.rgw.otp", 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "system_key": { 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "access_key": "", 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "secret_key": "" 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_pools": [ 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "key": "default-placement", 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "val": { 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "index_pool": "z.rgw.buckets.index", 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "storage_classes": { 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "STANDARD": { 2024-09-17T08:48:57.137 INFO:teuthology.orchestra.run.smithi027.stdout: "data_pool": "z.rgw.buckets.data" 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: "index_type": 0 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "acd7501c-bb75-4ce7-b40c-cb4e10f79740", 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout: "notif_pool": "z.rgw.log:notif" 2024-09-17T08:48:57.138 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:48:57.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-09-17T08:48:58.099 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:48:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:56.801002+0000 mgr.y (mgr.14144) 204 : cluster [DBG] pgmap v186: 33 pgs: 33 active+clean; 680 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 2.0 KiB/s wr, 2 op/s 2024-09-17T08:48:58.099 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:48:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:56.801002+0000 mgr.y (mgr.14144) 204 : cluster [DBG] pgmap v186: 33 pgs: 33 active+clean; 680 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 2.0 KiB/s wr, 2 op/s 2024-09-17T08:48:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:48:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:56.801002+0000 mgr.y (mgr.14144) 204 : cluster [DBG] pgmap v186: 33 pgs: 33 active+clean; 680 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 2.0 KiB/s wr, 2 op/s 2024-09-17T08:49:00.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:49:00.282Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.000920669s 2024-09-17T08:49:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:48:58.801401+0000 mgr.y (mgr.14144) 205 : cluster [DBG] pgmap v187: 33 pgs: 33 active+clean; 772 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-09-17T08:49:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:48:58.801401+0000 mgr.y (mgr.14144) 205 : cluster [DBG] pgmap v187: 33 pgs: 33 active+clean; 772 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-09-17T08:49:00.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:48:58.801401+0000 mgr.y (mgr.14144) 205 : cluster [DBG] pgmap v187: 33 pgs: 33 active+clean; 772 B data, 43 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-09-17T08:49:01.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:00.041031+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-17T08:49:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:00.043021+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:00.047732+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:00.041031+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-17T08:49:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:00.043021+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:00.047732+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:01.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:00.041031+0000 mon.a (mon.0) 484 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-17T08:49:01.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:00.043021+0000 mon.c (mon.1) 19 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:01.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:00.047732+0000 mon.a (mon.0) 485 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-09-17T08:49:02.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:00.801812+0000 mgr.y (mgr.14144) 206 : cluster [DBG] pgmap v189: 65 pgs: 5 creating+peering, 33 active+clean, 27 unknown; 1.8 KiB data, 43 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-09-17T08:49:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:01.042584+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T08:49:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:01.042723+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-17T08:49:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:00.801812+0000 mgr.y (mgr.14144) 206 : cluster [DBG] pgmap v189: 65 pgs: 5 creating+peering, 33 active+clean, 27 unknown; 1.8 KiB data, 43 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-09-17T08:49:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:01.042584+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T08:49:02.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:01.042723+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-17T08:49:02.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:00.801812+0000 mgr.y (mgr.14144) 206 : cluster [DBG] pgmap v189: 65 pgs: 5 creating+peering, 33 active+clean, 27 unknown; 1.8 KiB data, 43 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.8 KiB/s wr, 3 op/s 2024-09-17T08:49:02.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:01.042584+0000 mon.a (mon.0) 486 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-09-17T08:49:02.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:01.042723+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-17T08:49:03.381 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:02.061468+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-17T08:49:03.381 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:02.064605+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.381 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:02.072498+0000 mon.a (mon.0) 489 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.381 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:03.063257+0000 mon.a (mon.0) 490 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T08:49:03.381 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:03.063353+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-17T08:49:03.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:02.061468+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-17T08:49:03.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:02.064605+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:02.072498+0000 mon.a (mon.0) 489 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:03.063257+0000 mon.a (mon.0) 490 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:03.063353+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:02.061468+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:02.064605+0000 mon.c (mon.1) 20 : audit [INF] from='client.? 172.21.15.27:0/990896134' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:02.072498+0000 mon.a (mon.0) 489 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:03.063257+0000 mon.a (mon.0) 490 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-09-17T08:49:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:03.063353+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-17T08:49:04.383 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:02.802351+0000 mgr.y (mgr.14144) 207 : cluster [DBG] pgmap v192: 97 pgs: 21 creating+peering, 43 active+clean, 33 unknown; 1.8 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-17T08:49:04.384 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:04.062462+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-17T08:49:04.384 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:04.066121+0000 mon.a (mon.0) 493 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T08:49:04.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:02.802351+0000 mgr.y (mgr.14144) 207 : cluster [DBG] pgmap v192: 97 pgs: 21 creating+peering, 43 active+clean, 33 unknown; 1.8 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-17T08:49:04.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:04.062462+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-17T08:49:04.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:04.066121+0000 mon.a (mon.0) 493 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T08:49:04.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:02.802351+0000 mgr.y (mgr.14144) 207 : cluster [DBG] pgmap v192: 97 pgs: 21 creating+peering, 43 active+clean, 33 unknown; 1.8 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-17T08:49:04.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:04.062462+0000 mon.a (mon.0) 492 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-17T08:49:04.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:04.066121+0000 mon.a (mon.0) 493 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-09-17T08:49:06.028 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:05 smithi105 podman[36376]: 2024-09-17 08:49:05.936021639 +0000 UTC m=+0.127697910 container create 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.buildah.version=1.19.8, org.label-schema.license=GPLv2, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, description=Ceph Grafana Container) 2024-09-17T08:49:06.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:04.802877+0000 mgr.y (mgr.14144) 208 : cluster [DBG] pgmap v195: 129 pgs: 21 creating+peering, 63 active+clean, 45 unknown; 5.6 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 28 op/s 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:05.063939+0000 mon.a (mon.0) 494 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:05.064225+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:05.073773+0000 mon.a (mon.0) 496 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:04.802877+0000 mgr.y (mgr.14144) 208 : cluster [DBG] pgmap v195: 129 pgs: 21 creating+peering, 63 active+clean, 45 unknown; 5.6 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 28 op/s 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:05.063939+0000 mon.a (mon.0) 494 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:05.064225+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-17T08:49:06.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:05.073773+0000 mon.a (mon.0) 496 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T08:49:06.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:04.802877+0000 mgr.y (mgr.14144) 208 : cluster [DBG] pgmap v195: 129 pgs: 21 creating+peering, 63 active+clean, 45 unknown; 5.6 KiB data, 44 MiB used, 715 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 KiB/s wr, 28 op/s 2024-09-17T08:49:06.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:05.063939+0000 mon.a (mon.0) 494 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-09-17T08:49:06.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:05.064225+0000 mon.a (mon.0) 495 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-17T08:49:06.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:05.073773+0000 mon.a (mon.0) 496 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-09-17T08:49:06.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 podman[36376]: 2024-09-17 08:49:06.146286929 +0000 UTC m=+0.337963155 container init 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, description=Ceph Grafana Container, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, io.buildah.version=1.19.8) 2024-09-17T08:49:06.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 podman[36376]: 2024-09-17 08:49:06.149313835 +0000 UTC m=+0.340990062 container start 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, description=Ceph Grafana Container, org.label-schema.build-date=20201204, io.buildah.version=1.19.8, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Paul Cuzner ) 2024-09-17T08:49:06.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Starting Grafana" logger=server version=6.7.4 commit=8e44bbc5f5 branch=HEAD compiled=2020-05-26T17:35:38+0000 2024-09-17T08:49:06.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-17T08:49:06.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="App mode production" logger=settings 2024-09-17T08:49:06.529 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing SqlStore" logger=server 2024-09-17T08:49:06.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-17T08:49:06.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Starting DB migration" logger=migrator 2024-09-17T08:49:06.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-09-17T08:49:06.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-09-17T08:49:06.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-09-17T08:49:06.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-17T08:49:06.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-09-17T08:49:06.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-09-17T08:49:06.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-09-17T08:49:06.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-09-17T08:49:06.538 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-09-17T08:49:06.539 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-09-17T08:49:06.539 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-09-17T08:49:06.539 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-09-17T08:49:06.540 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-09-17T08:49:06.541 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-09-17T08:49:06.542 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-09-17T08:49:06.543 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-09-17T08:49:06.544 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-17T08:49:06.545 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-17T08:49:06.546 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-17T08:49:06.547 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-17T08:49:06.547 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-17T08:49:06.547 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-17T08:49:06.547 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-17T08:49:06.547 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-17T08:49:06.548 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-17T08:49:06.549 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-17T08:49:06.550 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-17T08:49:06.551 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-17T08:49:06.552 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-17T08:49:06.553 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 bash[36376]: 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 systemd[1]: Started Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Created default admin" logger=sqlstore user=admin 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing HTTPServer" logger=server 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing BackendPluginManager" logger=server 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing PluginManager" logger=server 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Starting plugin search" logger=plugins 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Registering plugin" logger=plugins name="Pie Chart" 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Registering plugin" logger=plugins name="Status Panel" 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing HooksService" logger=server 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing OSSLicensingService" logger=server 2024-09-17T08:49:06.554 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing InternalMetricsService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing RemoteCache" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing RenderingService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing AlertEngine" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing QuotaService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing ServerLockService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing UserAuthTokenService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing DatasourceCacheService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing LoginService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing SearchService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing TracingService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing UsageStatsService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing CleanUpService" logger=server 2024-09-17T08:49:06.555 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing NotificationService" logger=server 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing provisioningServiceImpl" logger=server 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Backend rendering via phantomJS" logger=rendering renderer=phantomJS 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=warn msg="phantomJS is deprecated and will be removed in a future release. You should consider migrating from phantomJS to grafana-image-renderer plugin. Read more at https://grafana.com/docs/grafana/latest/administration/image_rendering/" logger=rendering renderer=phantomJS 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="Initializing Stream Manager" 2024-09-17T08:49:06.556 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:49:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:49:06+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "0cde7150-159c-4a59-a69c-a24f456bf1a8", 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "epoch": 1, 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "predecessor_uuid": "3e017442-65e2-4dc5-8a63-dcb415d51154", 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_status": [], 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "period_map": { 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "0cde7150-159c-4a59-a69c-a24f456bf1a8", 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: "zonegroups": [ 2024-09-17T08:49:07.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "32cf48b3-07cf-42cc-a6a7-af55b3fac290", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "api_name": "default", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "is_master": "true", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames": [], 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "hostnames_s3website": [], 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "5c0596db-6a9c-4cdf-bf18-538d78a96a23", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "zones": [ 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "id": "5c0596db-6a9c-4cdf-bf18-538d78a96a23", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "z", 2024-09-17T08:49:07.159 INFO:teuthology.orchestra.run.smithi027.stdout: "endpoints": [], 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "log_meta": "false", 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "log_data": "false", 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "bucket_index_max_shards": 11, 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "read_only": "false", 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "tier_type": "", 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_from_all": "true", 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_from": [], 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "redirect_zone": "" 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "placement_targets": [ 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-09-17T08:49:07.160 INFO:teuthology.orchestra.run.smithi027.stdout: "name": "default-placement", 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "tags": [], 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "storage_classes": [ 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "STANDARD" 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "default_placement": "default-placement", 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "acd7501c-bb75-4ce7-b40c-cb4e10f79740", 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "sync_policy": { 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: "groups": [] 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.161 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "short_zone_ids": [ 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "key": "5c0596db-6a9c-4cdf-bf18-538d78a96a23", 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "val": 305607494 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zonegroup": "32cf48b3-07cf-42cc-a6a7-af55b3fac290", 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "master_zone": "5c0596db-6a9c-4cdf-bf18-538d78a96a23", 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "period_config": { 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "bucket_quota": { 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false, 2024-09-17T08:49:07.162 INFO:teuthology.orchestra.run.smithi027.stdout: "check_on_raw": false, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size": -1, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size_kb": 0, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_objects": -1 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "user_quota": { 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "enabled": false, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "check_on_raw": false, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size": -1, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_size_kb": 0, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "max_objects": -1 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:49:07.163 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_id": "acd7501c-bb75-4ce7-b40c-cb4e10f79740", 2024-09-17T08:49:07.164 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_name": "r", 2024-09-17T08:49:07.164 INFO:teuthology.orchestra.run.smithi027.stdout: "realm_epoch": 2 2024-09-17T08:49:07.164 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:06.075818+0000 mon.a (mon.0) 497 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:06.075914+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:06.076743+0000 mon.a (mon.0) 499 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:06.314733+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:06.315800+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:07.415 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:07.075084+0000 mon.a (mon.0) 502 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:07.075150+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:06.075818+0000 mon.a (mon.0) 497 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:06.075914+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:06.076743+0000 mon.a (mon.0) 499 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:06.314733+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:06.315800+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:07.075084+0000 mon.a (mon.0) 502 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T08:49:07.416 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:07.075150+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-17T08:49:07.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:06.075818+0000 mon.a (mon.0) 497 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:06.075914+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:06.076743+0000 mon.a (mon.0) 499 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:06.314733+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:06.315800+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:07.075084+0000 mon.a (mon.0) 502 : audit [INF] from='client.? 172.21.15.27:0/1510918130' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-09-17T08:49:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:07.075150+0000 mon.a (mon.0) 503 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-17T08:49:07.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-09-17T08:49:08.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:06.803224+0000 mgr.y (mgr.14144) 209 : cluster [DBG] pgmap v198: 129 pgs: 19 creating+peering, 86 active+clean, 24 unknown; 12 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 16 KiB/s wr, 68 op/s 2024-09-17T08:49:08.455 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:06.803224+0000 mgr.y (mgr.14144) 209 : cluster [DBG] pgmap v198: 129 pgs: 19 creating+peering, 86 active+clean, 24 unknown; 12 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 16 KiB/s wr, 68 op/s 2024-09-17T08:49:08.733 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:06.803224+0000 mgr.y (mgr.14144) 209 : cluster [DBG] pgmap v198: 129 pgs: 19 creating+peering, 86 active+clean, 24 unknown; 12 KiB data, 45 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 16 KiB/s wr, 68 op/s 2024-09-17T08:49:10.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:08.803793+0000 mgr.y (mgr.14144) 210 : cluster [DBG] pgmap v200: 129 pgs: 3 creating+peering, 122 active+clean, 4 unknown; 25 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 26 KiB/s wr, 103 op/s 2024-09-17T08:49:10.638 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled rgw.foo update... 2024-09-17T08:49:10.639 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:08.803793+0000 mgr.y (mgr.14144) 210 : cluster [DBG] pgmap v200: 129 pgs: 3 creating+peering, 122 active+clean, 4 unknown; 25 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 26 KiB/s wr, 103 op/s 2024-09-17T08:49:10.639 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:08.803793+0000 mgr.y (mgr.14144) 210 : cluster [DBG] pgmap v200: 129 pgs: 3 creating+peering, 122 active+clean, 4 unknown; 25 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 26 KiB/s wr, 103 op/s 2024-09-17T08:49:11.482 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph osd pool create foo' 2024-09-17T08:49:11.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:10.631528+0000 mgr.y (mgr.14144) 211 : audit [DBG] from='client.14631 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:10.632495+0000 mgr.y (mgr.14144) 212 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:10.638244+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:10.804327+0000 mgr.y (mgr.14144) 213 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 27 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 24 KiB/s wr, 92 op/s 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:10.885657+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:10.892728+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:10.631528+0000 mgr.y (mgr.14144) 211 : audit [DBG] from='client.14631 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:10.632495+0000 mgr.y (mgr.14144) 212 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:11.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:10.638244+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:11.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:10.804327+0000 mgr.y (mgr.14144) 213 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 27 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 24 KiB/s wr, 92 op/s 2024-09-17T08:49:11.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:10.885657+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:49:11.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:10.892728+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:10.631528+0000 mgr.y (mgr.14144) 211 : audit [DBG] from='client.14631 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:10.632495+0000 mgr.y (mgr.14144) 212 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:10.638244+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:10.804327+0000 mgr.y (mgr.14144) 213 : cluster [DBG] pgmap v201: 129 pgs: 129 active+clean; 27 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 24 KiB/s wr, 92 op/s 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:10.885657+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:49:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:10.892728+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:49:12.932 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:11.835564+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:12.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:11.835564+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:13.233 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:11.835564+0000 mon.a (mon.0) 507 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:14.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:12.804718+0000 mgr.y (mgr.14144) 214 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 15 KiB/s wr, 59 op/s 2024-09-17T08:49:14.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:12.804718+0000 mgr.y (mgr.14144) 214 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 15 KiB/s wr, 59 op/s 2024-09-17T08:49:14.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:12.804718+0000 mgr.y (mgr.14144) 214 : cluster [DBG] pgmap v202: 129 pgs: 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 15 KiB/s wr, 59 op/s 2024-09-17T08:49:14.614 INFO:teuthology.orchestra.run.smithi027.stderr:pool 'foo' created 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:13.634165+0000 mon.a (mon.0) 508 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.081146+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.302901+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.310801+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.317437+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:14.317968+0000 mgr.y (mgr.14144) 215 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.323756+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.324344+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.330073+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.335039+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.335703+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:14.336426+0000 mgr.y (mgr.14144) 216 : cephadm [INF] Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:14.605053+0000 mon.a (mon.0) 518 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:14.605169+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:13.634165+0000 mon.a (mon.0) 508 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.081146+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.302901+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.310801+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.317437+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:14.317968+0000 mgr.y (mgr.14144) 215 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.323756+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.324344+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.330073+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.335039+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.335703+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:14.336426+0000 mgr.y (mgr.14144) 216 : cephadm [INF] Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:14.605053+0000 mon.a (mon.0) 518 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T08:49:15.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:14.605169+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:13.634165+0000 mon.a (mon.0) 508 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.081146+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.302901+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.310801+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.317437+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:14.317968+0000 mgr.y (mgr.14144) 215 : cephadm [INF] Saving service rgw.foo spec with placement count:2 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.323756+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.324344+0000 mon.a (mon.0) 514 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.330073+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.335039+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.335703+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:14.336426+0000 mgr.y (mgr.14144) 216 : cephadm [INF] Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T08:49:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:14.605053+0000 mon.a (mon.0) 518 : audit [INF] from='client.? 172.21.15.27:0/512866635' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-09-17T08:49:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:14.605169+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-17T08:49:15.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'rbd pool init foo' 2024-09-17T08:49:16.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:14.805056+0000 mgr.y (mgr.14144) 217 : cluster [DBG] pgmap v204: 161 pgs: 32 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 13 KiB/s wr, 49 op/s 2024-09-17T08:49:16.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:15.608683+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-17T08:49:16.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:14.805056+0000 mgr.y (mgr.14144) 217 : cluster [DBG] pgmap v204: 161 pgs: 32 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 13 KiB/s wr, 49 op/s 2024-09-17T08:49:16.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:15.608683+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-17T08:49:16.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:14.805056+0000 mgr.y (mgr.14144) 217 : cluster [DBG] pgmap v204: 161 pgs: 32 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 13 KiB/s wr, 49 op/s 2024-09-17T08:49:16.465 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:15.608683+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-17T08:49:17.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:16.486044+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:16.486884+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:17.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:16.489863+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:17.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:16.494295+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:16.494945+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:17.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:16.495482+0000 mgr.y (mgr.14144) 218 : cephadm [INF] Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T08:49:17.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:16.611361+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-17T08:49:17.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:16.486044+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:16.486884+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:17.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:16.489863+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:17.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:16.494295+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:16.494945+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:17.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:16.495482+0000 mgr.y (mgr.14144) 218 : cephadm [INF] Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T08:49:17.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:16.611361+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-17T08:49:17.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:16.486044+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:16.486884+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T08:49:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:16.489863+0000 mon.a (mon.0) 523 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T08:49:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:16.494295+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:16.494945+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:16.495482+0000 mgr.y (mgr.14144) 218 : cephadm [INF] Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T08:49:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:16.611361+0000 mon.a (mon.0) 526 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-17T08:49:18.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:16.805340+0000 mgr.y (mgr.14144) 219 : cluster [DBG] pgmap v207: 161 pgs: 9 creating+peering, 23 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:49:18.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:17.518631+0000 mon.a (mon.0) 527 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T08:49:18.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:16.805340+0000 mgr.y (mgr.14144) 219 : cluster [DBG] pgmap v207: 161 pgs: 9 creating+peering, 23 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:49:18.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:17.518631+0000 mon.a (mon.0) 527 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T08:49:18.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:16.805340+0000 mgr.y (mgr.14144) 219 : cluster [DBG] pgmap v207: 161 pgs: 9 creating+peering, 23 unknown, 129 active+clean; 27 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:49:18.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:17.518631+0000 mon.a (mon.0) 527 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-09-17T08:49:19.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:18.495111+0000 mon.a (mon.0) 528 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T08:49:19.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:18.495211+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-17T08:49:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:18.495111+0000 mon.a (mon.0) 528 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T08:49:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:18.495211+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-17T08:49:19.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:18.495111+0000 mon.a (mon.0) 528 : audit [INF] from='client.? 172.21.15.27:0/950310365' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-09-17T08:49:19.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:18.495211+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-17T08:49:20.500 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:18.805667+0000 mgr.y (mgr.14144) 220 : cluster [DBG] pgmap v209: 161 pgs: 9 creating+peering, 3 unknown, 149 active+clean; 27 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 131 KiB/s rd, 0 B/s wr, 270 op/s 2024-09-17T08:49:20.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:19.500003+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-17T08:49:20.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:19.553291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:20.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:19.553979+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:20.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:18.805667+0000 mgr.y (mgr.14144) 220 : cluster [DBG] pgmap v209: 161 pgs: 9 creating+peering, 3 unknown, 149 active+clean; 27 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 131 KiB/s rd, 0 B/s wr, 270 op/s 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:19.500003+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:19.553291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:19.553979+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:18.805667+0000 mgr.y (mgr.14144) 220 : cluster [DBG] pgmap v209: 161 pgs: 9 creating+peering, 3 unknown, 149 active+clean; 27 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 131 KiB/s rd, 0 B/s wr, 270 op/s 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:19.500003+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:19.553291+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:19.553979+0000 mon.a (mon.0) 532 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:21.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch apply iscsi foo u p' 2024-09-17T08:49:21.672 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:20.506405+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-17T08:49:21.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:20.506405+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-17T08:49:22.036 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:20.506405+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-17T08:49:22.618 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:20.806362+0000 mgr.y (mgr.14144) 221 : cluster [DBG] pgmap v212: 161 pgs: 161 active+clean; 27 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 243 B/s wr, 598 op/s 2024-09-17T08:49:22.904 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:20.806362+0000 mgr.y (mgr.14144) 221 : cluster [DBG] pgmap v212: 161 pgs: 161 active+clean; 27 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 243 B/s wr, 598 op/s 2024-09-17T08:49:22.904 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:20.806362+0000 mgr.y (mgr.14144) 221 : cluster [DBG] pgmap v212: 161 pgs: 161 active+clean; 27 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 298 KiB/s rd, 243 B/s wr, 598 op/s 2024-09-17T08:49:23.508 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled iscsi.foo update... 2024-09-17T08:49:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:22.806909+0000 mgr.y (mgr.14144) 222 : cluster [DBG] pgmap v213: 161 pgs: 161 active+clean; 27 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 268 KiB/s rd, 511 B/s wr, 523 op/s 2024-09-17T08:49:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:22.997622+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:23.504762+0000 mgr.y (mgr.14144) 223 : audit [DBG] from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:23.505238+0000 mgr.y (mgr.14144) 224 : cephadm [INF] Saving service iscsi.foo spec with placement count:1 2024-09-17T08:49:24.253 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:23.508558+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:22.806909+0000 mgr.y (mgr.14144) 222 : cluster [DBG] pgmap v213: 161 pgs: 161 active+clean; 27 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 268 KiB/s rd, 511 B/s wr, 523 op/s 2024-09-17T08:49:24.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:22.997622+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:23.504762+0000 mgr.y (mgr.14144) 223 : audit [DBG] from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:24.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:23.505238+0000 mgr.y (mgr.14144) 224 : cephadm [INF] Saving service iscsi.foo spec with placement count:1 2024-09-17T08:49:24.254 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:23.508558+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:22.806909+0000 mgr.y (mgr.14144) 222 : cluster [DBG] pgmap v213: 161 pgs: 161 active+clean; 27 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 268 KiB/s rd, 511 B/s wr, 523 op/s 2024-09-17T08:49:24.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:22.997622+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:23.504762+0000 mgr.y (mgr.14144) 223 : audit [DBG] from='client.14691 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:49:24.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:23.505238+0000 mgr.y (mgr.14144) 224 : cephadm [INF] Saving service iscsi.foo spec with placement count:1 2024-09-17T08:49:24.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:23.508558+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:24.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T08:49:26.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:24.807547+0000 mgr.y (mgr.14144) 225 : cluster [DBG] pgmap v214: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 648 B/s wr, 465 op/s 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:25.164476+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:25.170350+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:25.172191+0000 mgr.y (mgr.14144) 226 : cephadm [INF] Checking pool "foo" exists for service iscsi.foo 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:25.178175+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:25.183596+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:25.187102+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:26.460 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:25.187618+0000 mgr.y (mgr.14144) 227 : cephadm [INF] Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:49:26.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:24.807547+0000 mgr.y (mgr.14144) 225 : cluster [DBG] pgmap v214: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 648 B/s wr, 465 op/s 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:25.164476+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:25.170350+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:25.172191+0000 mgr.y (mgr.14144) 226 : cephadm [INF] Checking pool "foo" exists for service iscsi.foo 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:25.178175+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:25.183596+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:25.187102+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:25.187618+0000 mgr.y (mgr.14144) 227 : cephadm [INF] Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:24.807547+0000 mgr.y (mgr.14144) 225 : cluster [DBG] pgmap v214: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 648 B/s wr, 465 op/s 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:25.164476+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:25.170350+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:25.172191+0000 mgr.y (mgr.14144) 226 : cephadm [INF] Checking pool "foo" exists for service iscsi.foo 2024-09-17T08:49:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:25.178175+0000 mon.a (mon.0) 538 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:49:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:25.183596+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-09-17T08:49:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:25.187102+0000 mon.a (mon.0) 540 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:49:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:25.187618+0000 mgr.y (mgr.14144) 227 : cephadm [INF] Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:49:28.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:26.808010+0000 mgr.y (mgr.14144) 228 : cluster [DBG] pgmap v215: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 199 KiB/s rd, 511 B/s wr, 363 op/s 2024-09-17T08:49:28.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:27.175041+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e15: y(active, since 5m), standbys: x 2024-09-17T08:49:28.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:27.610047+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-17T08:49:28.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:27.882026+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:28.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:27.883015+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:28.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:26.808010+0000 mgr.y (mgr.14144) 228 : cluster [DBG] pgmap v215: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 199 KiB/s rd, 511 B/s wr, 363 op/s 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:27.175041+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e15: y(active, since 5m), standbys: x 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:27.610047+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:27.882026+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:27.883015+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:26.808010+0000 mgr.y (mgr.14144) 228 : cluster [DBG] pgmap v215: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 199 KiB/s rd, 511 B/s wr, 363 op/s 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:27.175041+0000 mon.a (mon.0) 541 : cluster [DBG] mgrmap e15: y(active, since 5m), standbys: x 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:27.610047+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:27.882026+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:28.480 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:27.883015+0000 mon.a (mon.0) 544 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:29.464 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:28.315714+0000 mon.b (mon.2) 23 : audit [DBG] from='client.? 172.21.15.105:0/1057641330' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:49:29.464 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:28.315714+0000 mon.b (mon.2) 23 : audit [DBG] from='client.? 172.21.15.105:0/1057641330' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:49:29.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:28.315714+0000 mon.b (mon.2) 23 : audit [DBG] from='client.? 172.21.15.105:0/1057641330' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:49:30.294 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:28.808451+0000 mgr.y (mgr.14144) 229 : cluster [DBG] pgmap v217: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 369 B/s wr, 253 op/s 2024-09-17T08:49:30.294 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:28.808451+0000 mgr.y (mgr.14144) 229 : cluster [DBG] pgmap v217: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 369 B/s wr, 253 op/s 2024-09-17T08:49:30.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:28.808451+0000 mgr.y (mgr.14144) 229 : cluster [DBG] pgmap v217: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 143 KiB/s rd, 369 B/s wr, 253 op/s 2024-09-17T08:49:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:30.774550+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:30.808894+0000 mgr.y (mgr.14144) 230 : cluster [DBG] pgmap v218: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 307 B/s wr, 153 op/s 2024-09-17T08:49:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:31.723189+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:31.730103+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:30.774550+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:30.808894+0000 mgr.y (mgr.14144) 230 : cluster [DBG] pgmap v218: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 307 B/s wr, 153 op/s 2024-09-17T08:49:32.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:31.723189+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.037 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:31.730103+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.037 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:30.774550+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:30.808894+0000 mgr.y (mgr.14144) 230 : cluster [DBG] pgmap v218: 161 pgs: 161 active+clean; 27 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 307 B/s wr, 153 op/s 2024-09-17T08:49:32.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:31.723189+0000 mon.a (mon.0) 546 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:32.038 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:31.730103+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:33.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:31.732841+0000 mgr.y (mgr.14144) 231 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:49:33.166 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:31.735702+0000 mgr.y (mgr.14144) 232 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:49:33.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:31.732841+0000 mgr.y (mgr.14144) 231 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:49:33.167 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:31.735702+0000 mgr.y (mgr.14144) 232 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:49:33.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:31.732841+0000 mgr.y (mgr.14144) 231 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:49:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:31.735702+0000 mgr.y (mgr.14144) 232 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:49:34.067 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:32.809594+0000 mgr.y (mgr.14144) 233 : cluster [DBG] pgmap v219: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 204 B/s wr, 92 op/s 2024-09-17T08:49:34.067 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:32.809594+0000 mgr.y (mgr.14144) 233 : cluster [DBG] pgmap v219: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 204 B/s wr, 92 op/s 2024-09-17T08:49:34.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:32.809594+0000 mgr.y (mgr.14144) 233 : cluster [DBG] pgmap v219: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 204 B/s wr, 92 op/s 2024-09-17T08:49:34.958 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:34 smithi027 systemd[1]: Stopping Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:49:35.257 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[41755]: level=info ts=2024-09-17T08:49:34.958Z caller=main.go:536 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T08:49:35.257 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:34 smithi027 podman[44992]: 2024-09-17 08:49:34.979087768 +0000 UTC m=+0.146075221 container died 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:35.334 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[44992]: 2024-09-17 08:49:35.33292693 +0000 UTC m=+0.499914372 container cleanup 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:35.626 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[44992]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a 2024-09-17T08:49:35.898 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:34.810111+0000 mgr.y (mgr.14144) 234 : cluster [DBG] pgmap v220: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 102 B/s wr, 14 op/s 2024-09-17T08:49:35.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:34.810111+0000 mgr.y (mgr.14144) 234 : cluster [DBG] pgmap v220: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 102 B/s wr, 14 op/s 2024-09-17T08:49:35.899 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[45005]: 2024-09-17 08:49:35.625860301 +0000 UTC m=+0.644852848 container remove 145107f7b65c7300965146aa31ef8cffbfaafe707d736de93339052ec5784ee6 (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:35.899 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[45005]: 2024-09-17 08:49:35.701564579 +0000 UTC m=+0.720557125 volume remove 2074cb979df578cb7fcc985709b6ea2eac9e2a8cc6099616bdd1e98cc70beff8 2024-09-17T08:49:35.899 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T08:49:35.899 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 systemd[1]: Stopped Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:49:35.899 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 systemd[1]: Starting Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:49:36.257 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[45051]: 2024-09-17 08:49:35.801137045 +0000 UTC m=+0.017794918 image pull 0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0 docker.io/prom/alertmanager:v0.20.0 2024-09-17T08:49:36.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[45051]: 2024-09-17 08:49:35.902364903 +0000 UTC m=+0.119022808 volume create 8a28b5adde083a1954b8c0a5a413d0938036d8e6535c578158e4fb8a05e243e7 2024-09-17T08:49:36.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:35 smithi027 podman[45051]: 2024-09-17 08:49:35.986551349 +0000 UTC m=+0.203209258 container create 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:34.810111+0000 mgr.y (mgr.14144) 234 : cluster [DBG] pgmap v220: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 8.4 KiB/s rd, 102 B/s wr, 14 op/s 2024-09-17T08:49:36.757 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 podman[45051]: 2024-09-17 08:49:36.372093805 +0000 UTC m=+0.588751669 container init 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 podman[45051]: 2024-09-17 08:49:36.375665772 +0000 UTC m=+0.592323636 container start 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.399Z caller=main.go:231 msg="Starting Alertmanager" version="(version=0.20.0, branch=HEAD, revision=f74be0400a6243d10bb53812d6fa408ad71ff32d)" 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.399Z caller=main.go:232 build_context="(go=go1.13.5, user=root@00c3106655f8, date=20191211-14:13:14)" 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.400Z caller=cluster.go:161 component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.401Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.421Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.421Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:36.424Z caller=main.go:497 msg=Listening address=:9093 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 bash[45051]: 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a 2024-09-17T08:49:36.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:36 smithi027 systemd[1]: Started Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:36.478024+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:36.478972+0000 mgr.y (mgr.14144) 235 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:36.481218+0000 mgr.y (mgr.14144) 236 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:36.478024+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:36.478972+0000 mgr.y (mgr.14144) 235 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:49:37.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:36.481218+0000 mgr.y (mgr.14144) 236 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:49:37.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:36.478024+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:37.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:36.478972+0000 mgr.y (mgr.14144) 235 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:49:37.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:36.481218+0000 mgr.y (mgr.14144) 236 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:49:38.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:36.810653+0000 mgr.y (mgr.14144) 237 : cluster [DBG] pgmap v221: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 1 op/s 2024-09-17T08:49:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:36.810653+0000 mgr.y (mgr.14144) 237 : cluster [DBG] pgmap v221: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 1 op/s 2024-09-17T08:49:38.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:38.401Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00009682s 2024-09-17T08:49:38.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:36.810653+0000 mgr.y (mgr.14144) 237 : cluster [DBG] pgmap v221: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 102 B/s wr, 1 op/s 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 systemd[1]: Stopping Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=warn ts=2024-09-17T08:49:39.586Z caller=main.go:524 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:49:39.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[35650]: level=info ts=2024-09-17T08:49:39.586Z caller=main.go:761 msg="See you next time!" 2024-09-17T08:49:39.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 podman[38951]: 2024-09-17 08:49:39.596397484 +0000 UTC m=+0.126007479 container died cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:40.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 podman[38951]: 2024-09-17 08:49:39.977481968 +0000 UTC m=+0.507091964 container cleanup cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:40.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:39 smithi105 podman[38951]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a 2024-09-17T08:49:40.578 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:38.811270+0000 mgr.y (mgr.14144) 238 : cluster [DBG] pgmap v222: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 91 B/s wr, 1 op/s 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 podman[38964]: 2024-09-17 08:49:40.295301796 +0000 UTC m=+0.700617384 container remove cace2ccc26d6a1dc1264921f094ca30993e8abd2f3bac3aed6a78b6b2be59a89 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 podman[39004]: 2024-09-17 08:49:40.388369717 +0000 UTC m=+0.017688026 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:49:40.579 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 podman[39004]: 2024-09-17 08:49:40.496184637 +0000 UTC m=+0.125502980 container create d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:40.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:38.811270+0000 mgr.y (mgr.14144) 238 : cluster [DBG] pgmap v222: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 91 B/s wr, 1 op/s 2024-09-17T08:49:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:38.811270+0000 mgr.y (mgr.14144) 238 : cluster [DBG] pgmap v222: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 913 B/s rd, 91 B/s wr, 1 op/s 2024-09-17T08:49:41.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 podman[39004]: 2024-09-17 08:49:40.681161244 +0000 UTC m=+0.310479544 container init d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:41.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 podman[39004]: 2024-09-17 08:49:40.684828405 +0000 UTC m=+0.314146706 container start d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:49:41.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:49:41.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.715Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.716Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.716Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.718Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.719Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.719Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.719Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=1.128027ms 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.720Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.720Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.720Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:49:41.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.722Z caller=main.go:827 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:49:41.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:49:40.722Z caller=main.go:646 msg="Server is ready to receive web requests." 2024-09-17T08:49:41.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 bash[39004]: d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f 2024-09-17T08:49:41.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:49:40 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:49:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.775258+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.779507+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.779886+0000 mgr.y (mgr.14144) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.781029+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.781334+0000 mgr.y (mgr.14144) 240 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.787030+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.791965+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.792370+0000 mgr.y (mgr.14144) 241 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.799449+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.799785+0000 mgr.y (mgr.14144) 242 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.804310+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.808507+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.809043+0000 mgr.y (mgr.14144) 243 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:40.811796+0000 mgr.y (mgr.14144) 244 : cluster [DBG] pgmap v223: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.813530+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.813854+0000 mgr.y (mgr.14144) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.819367+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.823608+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.824351+0000 mgr.y (mgr.14144) 246 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:49:40.829825+0000 mgr.y (mgr.14144) 247 : cephadm [INF] Adding iSCSI gateway http://:@172.21.15.105:5000 to Dashboard 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.830104+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.830460+0000 mgr.y (mgr.14144) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.835908+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.838132+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.838839+0000 mgr.y (mgr.14144) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.845171+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:40.846486+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.775258+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.779507+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.779886+0000 mgr.y (mgr.14144) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.781029+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.781334+0000 mgr.y (mgr.14144) 240 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.787030+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.791965+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.792370+0000 mgr.y (mgr.14144) 241 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.799449+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.799785+0000 mgr.y (mgr.14144) 242 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.804310+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.808507+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.809043+0000 mgr.y (mgr.14144) 243 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:40.811796+0000 mgr.y (mgr.14144) 244 : cluster [DBG] pgmap v223: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.813530+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.775258+0000 mon.a (mon.0) 549 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.779507+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.779886+0000 mgr.y (mgr.14144) 239 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.781029+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.781334+0000 mgr.y (mgr.14144) 240 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027:9093"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.787030+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.791965+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.792370+0000 mgr.y (mgr.14144) 241 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.799449+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.799785+0000 mgr.y (mgr.14144) 242 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105:9095"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.804310+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.808507+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.809043+0000 mgr.y (mgr.14144) 243 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:40.811796+0000 mgr.y (mgr.14144) 244 : cluster [DBG] pgmap v223: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.813530+0000 mon.a (mon.0) 557 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.813854+0000 mgr.y (mgr.14144) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.819367+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.823608+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.824351+0000 mgr.y (mgr.14144) 246 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:49:40.829825+0000 mgr.y (mgr.14144) 247 : cephadm [INF] Adding iSCSI gateway http://:@172.21.15.105:5000 to Dashboard 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.830104+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.830460+0000 mgr.y (mgr.14144) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.835908+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.838132+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.838839+0000 mgr.y (mgr.14144) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.845171+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:40.846486+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.813854+0000 mgr.y (mgr.14144) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105:3000"}]: dispatch 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.819367+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.823608+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.824351+0000 mgr.y (mgr.14144) 246 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:49:40.829825+0000 mgr.y (mgr.14144) 247 : cephadm [INF] Adding iSCSI gateway http://:@172.21.15.105:5000 to Dashboard 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.830104+0000 mon.a (mon.0) 560 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.830460+0000 mgr.y (mgr.14144) 248 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.835908+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.838132+0000 mon.a (mon.0) 562 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.838839+0000 mgr.y (mgr.14144) 249 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi105"}]: dispatch 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.845171+0000 mon.a (mon.0) 563 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:42.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:40.846486+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:49:44.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:42.812248+0000 mgr.y (mgr.14144) 250 : cluster [DBG] pgmap v224: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:44.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:43.560967+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:44.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:42.812248+0000 mgr.y (mgr.14144) 250 : cluster [DBG] pgmap v224: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:44.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:43.560967+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:44.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:42.812248+0000 mgr.y (mgr.14144) 250 : cluster [DBG] pgmap v224: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T08:49:44.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:43.560967+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:44.802510+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:49:44.806567+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:44.812755+0000 mgr.y (mgr.14144) 251 : cluster [DBG] pgmap v225: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:46.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:44.802510+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:49:44.806567+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:44.812755+0000 mgr.y (mgr.14144) 251 : cluster [DBG] pgmap v225: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:46.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:44.802510+0000 mon.a (mon.0) 566 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:49:44.806567+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:49:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:44.812755+0000 mgr.y (mgr.14144) 251 : cluster [DBG] pgmap v225: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:46.757 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:49:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:49:46.402Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.000551596s 2024-09-17T08:49:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:46.813325+0000 mgr.y (mgr.14144) 252 : cluster [DBG] pgmap v226: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:46.813325+0000 mgr.y (mgr.14144) 252 : cluster [DBG] pgmap v226: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:46.813325+0000 mgr.y (mgr.14144) 252 : cluster [DBG] pgmap v226: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:48.814113+0000 mgr.y (mgr.14144) 253 : cluster [DBG] pgmap v227: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:48.814113+0000 mgr.y (mgr.14144) 253 : cluster [DBG] pgmap v227: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:50.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:48.814113+0000 mgr.y (mgr.14144) 253 : cluster [DBG] pgmap v227: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:52.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:50.814771+0000 mgr.y (mgr.14144) 254 : cluster [DBG] pgmap v228: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:50.814771+0000 mgr.y (mgr.14144) 254 : cluster [DBG] pgmap v228: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:52.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:50.814771+0000 mgr.y (mgr.14144) 254 : cluster [DBG] pgmap v228: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:53.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:52.815504+0000 mgr.y (mgr.14144) 255 : cluster [DBG] pgmap v229: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:53.973 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:52.815504+0000 mgr.y (mgr.14144) 255 : cluster [DBG] pgmap v229: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:54.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:52.815504+0000 mgr.y (mgr.14144) 255 : cluster [DBG] pgmap v229: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:49:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:54.816080+0000 mgr.y (mgr.14144) 256 : cluster [DBG] pgmap v230: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:56.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:54.816080+0000 mgr.y (mgr.14144) 256 : cluster [DBG] pgmap v230: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:54.816080+0000 mgr.y (mgr.14144) 256 : cluster [DBG] pgmap v230: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:56.816618+0000 mgr.y (mgr.14144) 257 : cluster [DBG] pgmap v231: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:56.816618+0000 mgr.y (mgr.14144) 257 : cluster [DBG] pgmap v231: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:49:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:56.816618+0000 mgr.y (mgr.14144) 257 : cluster [DBG] pgmap v231: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:00.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:49:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:49:58.817268+0000 mgr.y (mgr.14144) 258 : cluster [DBG] pgmap v232: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:49:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:49:58.817268+0000 mgr.y (mgr.14144) 258 : cluster [DBG] pgmap v232: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:49:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:49:58.817268+0000 mgr.y (mgr.14144) 258 : cluster [DBG] pgmap v232: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:00.000192+0000 mon.a (mon.0) 568 : cluster [INF] overall HEALTH_OK 2024-09-17T08:50:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:00.000192+0000 mon.a (mon.0) 568 : cluster [INF] overall HEALTH_OK 2024-09-17T08:50:01.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:00.000192+0000 mon.a (mon.0) 568 : cluster [INF] overall HEALTH_OK 2024-09-17T08:50:02.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:00.817818+0000 mgr.y (mgr.14144) 259 : cluster [DBG] pgmap v233: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:02.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:00.817818+0000 mgr.y (mgr.14144) 259 : cluster [DBG] pgmap v233: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:00.817818+0000 mgr.y (mgr.14144) 259 : cluster [DBG] pgmap v233: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:02.818500+0000 mgr.y (mgr.14144) 260 : cluster [DBG] pgmap v234: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:02.818500+0000 mgr.y (mgr.14144) 260 : cluster [DBG] pgmap v234: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:04.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:02.818500+0000 mgr.y (mgr.14144) 260 : cluster [DBG] pgmap v234: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:06.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:04.819041+0000 mgr.y (mgr.14144) 261 : cluster [DBG] pgmap v235: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:04.819041+0000 mgr.y (mgr.14144) 261 : cluster [DBG] pgmap v235: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:06.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:04.819041+0000 mgr.y (mgr.14144) 261 : cluster [DBG] pgmap v235: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:08.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:06.819604+0000 mgr.y (mgr.14144) 262 : cluster [DBG] pgmap v236: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:06.819604+0000 mgr.y (mgr.14144) 262 : cluster [DBG] pgmap v236: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:06.819604+0000 mgr.y (mgr.14144) 262 : cluster [DBG] pgmap v236: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:10.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:08.820379+0000 mgr.y (mgr.14144) 263 : cluster [DBG] pgmap v237: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:08.820379+0000 mgr.y (mgr.14144) 263 : cluster [DBG] pgmap v237: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:08.820379+0000 mgr.y (mgr.14144) 263 : cluster [DBG] pgmap v237: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.769029+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.769338+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.769560+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.769753+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.769029+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.769338+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.769560+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-17T08:50:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.769753+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-17T08:50:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.769029+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]: dispatch 2024-09-17T08:50:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.769338+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-17T08:50:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.769560+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-17T08:50:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.769753+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-17T08:50:12.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:10.820859+0000 mgr.y (mgr.14144) 264 : cluster [DBG] pgmap v238: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:12.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.891135+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-09-17T08:50:12.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.891244+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T08:50:12.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.891331+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-17T08:50:12.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.891424+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-17T08:50:12.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:10.891503+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-17T08:50:12.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.893159+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:50:12.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:10.900301+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:50:12.161 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:10.820859+0000 mgr.y (mgr.14144) 264 : cluster [DBG] pgmap v238: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:12.161 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.891135+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.891244+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.891331+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.891424+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:10.891503+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.893159+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:50:12.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:10.900301+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:50:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:10.820859+0000 mgr.y (mgr.14144) 264 : cluster [DBG] pgmap v238: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.891135+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [7, 2]}]': finished 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.891244+0000 mon.a (mon.0) 574 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.891331+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.891424+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:10.891503+0000 mon.a (mon.0) 577 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.893159+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:50:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:10.900301+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:50:13.181 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:11.892187+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-17T08:50:13.182 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:11.892187+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-17T08:50:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:11.892187+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-17T08:50:14.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:12.821513+0000 mgr.y (mgr.14144) 265 : cluster [DBG] pgmap v241: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T08:50:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:12.821513+0000 mgr.y (mgr.14144) 265 : cluster [DBG] pgmap v241: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T08:50:14.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:12.821513+0000 mgr.y (mgr.14144) 265 : cluster [DBG] pgmap v241: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T08:50:16.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:14.821998+0000 mgr.y (mgr.14144) 266 : cluster [DBG] pgmap v242: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:14.821998+0000 mgr.y (mgr.14144) 266 : cluster [DBG] pgmap v242: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:14.821998+0000 mgr.y (mgr.14144) 266 : cluster [DBG] pgmap v242: 161 pgs: 161 active+clean; 27 KiB data, 65 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:18.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:16.822536+0000 mgr.y (mgr.14144) 267 : cluster [DBG] pgmap v243: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:16.822536+0000 mgr.y (mgr.14144) 267 : cluster [DBG] pgmap v243: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:16.822536+0000 mgr.y (mgr.14144) 267 : cluster [DBG] pgmap v243: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:18.823259+0000 mgr.y (mgr.14144) 268 : cluster [DBG] pgmap v244: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:18.823259+0000 mgr.y (mgr.14144) 268 : cluster [DBG] pgmap v244: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:20.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:18.823259+0000 mgr.y (mgr.14144) 268 : cluster [DBG] pgmap v244: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 146 B/s, 0 objects/s recovering 2024-09-17T08:50:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:20.823781+0000 mgr.y (mgr.14144) 269 : cluster [DBG] pgmap v245: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 118 B/s, 0 objects/s recovering 2024-09-17T08:50:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:20.823781+0000 mgr.y (mgr.14144) 269 : cluster [DBG] pgmap v245: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 118 B/s, 0 objects/s recovering 2024-09-17T08:50:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:20.823781+0000 mgr.y (mgr.14144) 269 : cluster [DBG] pgmap v245: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s; 118 B/s, 0 objects/s recovering 2024-09-17T08:50:23.894 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:22.824259+0000 mgr.y (mgr.14144) 270 : cluster [DBG] pgmap v246: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 107 B/s, 0 objects/s recovering 2024-09-17T08:50:23.894 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:22.824259+0000 mgr.y (mgr.14144) 270 : cluster [DBG] pgmap v246: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 107 B/s, 0 objects/s recovering 2024-09-17T08:50:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:22.824259+0000 mgr.y (mgr.14144) 270 : cluster [DBG] pgmap v246: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s; 107 B/s, 0 objects/s recovering 2024-09-17T08:50:26.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:24.824843+0000 mgr.y (mgr.14144) 271 : cluster [DBG] pgmap v247: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97 B/s, 0 objects/s recovering 2024-09-17T08:50:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:24.824843+0000 mgr.y (mgr.14144) 271 : cluster [DBG] pgmap v247: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97 B/s, 0 objects/s recovering 2024-09-17T08:50:26.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:24.824843+0000 mgr.y (mgr.14144) 271 : cluster [DBG] pgmap v247: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 97 B/s, 0 objects/s recovering 2024-09-17T08:50:28.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:26.825517+0000 mgr.y (mgr.14144) 272 : cluster [DBG] pgmap v248: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:26.825517+0000 mgr.y (mgr.14144) 272 : cluster [DBG] pgmap v248: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:26.825517+0000 mgr.y (mgr.14144) 272 : cluster [DBG] pgmap v248: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:30.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:28.826109+0000 mgr.y (mgr.14144) 273 : cluster [DBG] pgmap v249: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:30.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:28.826109+0000 mgr.y (mgr.14144) 273 : cluster [DBG] pgmap v249: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:30.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:28.826109+0000 mgr.y (mgr.14144) 273 : cluster [DBG] pgmap v249: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:32.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:30.826654+0000 mgr.y (mgr.14144) 274 : cluster [DBG] pgmap v250: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:30.826654+0000 mgr.y (mgr.14144) 274 : cluster [DBG] pgmap v250: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:32.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:30.826654+0000 mgr.y (mgr.14144) 274 : cluster [DBG] pgmap v250: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:32.827149+0000 mgr.y (mgr.14144) 275 : cluster [DBG] pgmap v251: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:32.827149+0000 mgr.y (mgr.14144) 275 : cluster [DBG] pgmap v251: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:32.827149+0000 mgr.y (mgr.14144) 275 : cluster [DBG] pgmap v251: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:36.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:34.827685+0000 mgr.y (mgr.14144) 276 : cluster [DBG] pgmap v252: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:36.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:34.827685+0000 mgr.y (mgr.14144) 276 : cluster [DBG] pgmap v252: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:34.827685+0000 mgr.y (mgr.14144) 276 : cluster [DBG] pgmap v252: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:38.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:36.828156+0000 mgr.y (mgr.14144) 277 : cluster [DBG] pgmap v253: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:36.828156+0000 mgr.y (mgr.14144) 277 : cluster [DBG] pgmap v253: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:36.828156+0000 mgr.y (mgr.14144) 277 : cluster [DBG] pgmap v253: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:38.828773+0000 mgr.y (mgr.14144) 278 : cluster [DBG] pgmap v254: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:38.828773+0000 mgr.y (mgr.14144) 278 : cluster [DBG] pgmap v254: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:38.828773+0000 mgr.y (mgr.14144) 278 : cluster [DBG] pgmap v254: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:42.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:40.829272+0000 mgr.y (mgr.14144) 279 : cluster [DBG] pgmap v255: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:42.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:40.829272+0000 mgr.y (mgr.14144) 279 : cluster [DBG] pgmap v255: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:40.829272+0000 mgr.y (mgr.14144) 279 : cluster [DBG] pgmap v255: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:44.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:42.829771+0000 mgr.y (mgr.14144) 280 : cluster [DBG] pgmap v256: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:44.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:42.829771+0000 mgr.y (mgr.14144) 280 : cluster [DBG] pgmap v256: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:44.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:42.829771+0000 mgr.y (mgr.14144) 280 : cluster [DBG] pgmap v256: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:45.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:44.809715+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:50:45.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:44.809715+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:50:45.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:44.809715+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:50:46.161 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:44.830454+0000 mgr.y (mgr.14144) 281 : cluster [DBG] pgmap v257: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:44.830454+0000 mgr.y (mgr.14144) 281 : cluster [DBG] pgmap v257: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:46.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:44.830454+0000 mgr.y (mgr.14144) 281 : cluster [DBG] pgmap v257: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:48.196 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:46.831083+0000 mgr.y (mgr.14144) 282 : cluster [DBG] pgmap v258: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:46.831083+0000 mgr.y (mgr.14144) 282 : cluster [DBG] pgmap v258: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:48.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:46.831083+0000 mgr.y (mgr.14144) 282 : cluster [DBG] pgmap v258: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:48.615482+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:50:48.827610+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:48.831736+0000 mgr.y (mgr.14144) 283 : cluster [DBG] pgmap v259: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:48.615482+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:50:48.827610+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:48.831736+0000 mgr.y (mgr.14144) 283 : cluster [DBG] pgmap v259: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:50.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:48.615482+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:50:48.827610+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:50:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:48.831736+0000 mgr.y (mgr.14144) 283 : cluster [DBG] pgmap v259: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:52.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:50.832373+0000 mgr.y (mgr.14144) 284 : cluster [DBG] pgmap v260: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:50.832373+0000 mgr.y (mgr.14144) 284 : cluster [DBG] pgmap v260: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:52.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:50.832373+0000 mgr.y (mgr.14144) 284 : cluster [DBG] pgmap v260: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:54.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:52.832895+0000 mgr.y (mgr.14144) 285 : cluster [DBG] pgmap v261: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:52.832895+0000 mgr.y (mgr.14144) 285 : cluster [DBG] pgmap v261: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:54.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:52.832895+0000 mgr.y (mgr.14144) 285 : cluster [DBG] pgmap v261: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:50:56.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:54.833552+0000 mgr.y (mgr.14144) 286 : cluster [DBG] pgmap v262: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:56.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:54.833552+0000 mgr.y (mgr.14144) 286 : cluster [DBG] pgmap v262: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:54.833552+0000 mgr.y (mgr.14144) 286 : cluster [DBG] pgmap v262: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:58.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:56.834119+0000 mgr.y (mgr.14144) 287 : cluster [DBG] pgmap v263: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:56.834119+0000 mgr.y (mgr.14144) 287 : cluster [DBG] pgmap v263: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:50:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:56.834119+0000 mgr.y (mgr.14144) 287 : cluster [DBG] pgmap v263: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:00.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:50:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:50:58.834939+0000 mgr.y (mgr.14144) 288 : cluster [DBG] pgmap v264: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:50:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:50:58.834939+0000 mgr.y (mgr.14144) 288 : cluster [DBG] pgmap v264: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:50:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:50:58.834939+0000 mgr.y (mgr.14144) 288 : cluster [DBG] pgmap v264: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:02.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:00.835552+0000 mgr.y (mgr.14144) 289 : cluster [DBG] pgmap v265: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:00.835552+0000 mgr.y (mgr.14144) 289 : cluster [DBG] pgmap v265: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:02.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:00.835552+0000 mgr.y (mgr.14144) 289 : cluster [DBG] pgmap v265: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:02.836122+0000 mgr.y (mgr.14144) 290 : cluster [DBG] pgmap v266: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:02.836122+0000 mgr.y (mgr.14144) 290 : cluster [DBG] pgmap v266: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:02.836122+0000 mgr.y (mgr.14144) 290 : cluster [DBG] pgmap v266: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:06.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:04.836788+0000 mgr.y (mgr.14144) 291 : cluster [DBG] pgmap v267: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:04.836788+0000 mgr.y (mgr.14144) 291 : cluster [DBG] pgmap v267: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:04.836788+0000 mgr.y (mgr.14144) 291 : cluster [DBG] pgmap v267: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:08.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:06.837282+0000 mgr.y (mgr.14144) 292 : cluster [DBG] pgmap v268: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:08.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:06.837282+0000 mgr.y (mgr.14144) 292 : cluster [DBG] pgmap v268: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:06.837282+0000 mgr.y (mgr.14144) 292 : cluster [DBG] pgmap v268: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:10.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:08.837976+0000 mgr.y (mgr.14144) 293 : cluster [DBG] pgmap v269: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:08.837976+0000 mgr.y (mgr.14144) 293 : cluster [DBG] pgmap v269: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:08.837976+0000 mgr.y (mgr.14144) 293 : cluster [DBG] pgmap v269: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:12.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:10.838636+0000 mgr.y (mgr.14144) 294 : cluster [DBG] pgmap v270: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:51:10.901970+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:51:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:51:10.903415+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:51:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:10.838636+0000 mgr.y (mgr.14144) 294 : cluster [DBG] pgmap v270: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:51:10.901970+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:51:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:51:10.903415+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:51:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:10.838636+0000 mgr.y (mgr.14144) 294 : cluster [DBG] pgmap v270: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:51:10.901970+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:51:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:51:10.903415+0000 mon.a (mon.0) 585 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:51:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:12.839329+0000 mgr.y (mgr.14144) 295 : cluster [DBG] pgmap v271: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:12.839329+0000 mgr.y (mgr.14144) 295 : cluster [DBG] pgmap v271: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:12.839329+0000 mgr.y (mgr.14144) 295 : cluster [DBG] pgmap v271: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:16.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:14.840126+0000 mgr.y (mgr.14144) 296 : cluster [DBG] pgmap v272: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:14.840126+0000 mgr.y (mgr.14144) 296 : cluster [DBG] pgmap v272: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:14.840126+0000 mgr.y (mgr.14144) 296 : cluster [DBG] pgmap v272: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:16.840899+0000 mgr.y (mgr.14144) 297 : cluster [DBG] pgmap v273: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:16.840899+0000 mgr.y (mgr.14144) 297 : cluster [DBG] pgmap v273: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:16.840899+0000 mgr.y (mgr.14144) 297 : cluster [DBG] pgmap v273: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:18.841826+0000 mgr.y (mgr.14144) 298 : cluster [DBG] pgmap v274: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:18.841826+0000 mgr.y (mgr.14144) 298 : cluster [DBG] pgmap v274: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:20.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:18.841826+0000 mgr.y (mgr.14144) 298 : cluster [DBG] pgmap v274: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:22.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:20.842659+0000 mgr.y (mgr.14144) 299 : cluster [DBG] pgmap v275: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:20.842659+0000 mgr.y (mgr.14144) 299 : cluster [DBG] pgmap v275: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:20.842659+0000 mgr.y (mgr.14144) 299 : cluster [DBG] pgmap v275: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:24.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:22.843364+0000 mgr.y (mgr.14144) 300 : cluster [DBG] pgmap v276: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:22.843364+0000 mgr.y (mgr.14144) 300 : cluster [DBG] pgmap v276: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:24.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:22.843364+0000 mgr.y (mgr.14144) 300 : cluster [DBG] pgmap v276: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:26.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:24.844337+0000 mgr.y (mgr.14144) 301 : cluster [DBG] pgmap v277: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:24.844337+0000 mgr.y (mgr.14144) 301 : cluster [DBG] pgmap v277: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:24.844337+0000 mgr.y (mgr.14144) 301 : cluster [DBG] pgmap v277: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:28.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:26.845174+0000 mgr.y (mgr.14144) 302 : cluster [DBG] pgmap v278: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:26.845174+0000 mgr.y (mgr.14144) 302 : cluster [DBG] pgmap v278: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:26.845174+0000 mgr.y (mgr.14144) 302 : cluster [DBG] pgmap v278: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:28.845997+0000 mgr.y (mgr.14144) 303 : cluster [DBG] pgmap v279: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:28.845997+0000 mgr.y (mgr.14144) 303 : cluster [DBG] pgmap v279: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:30.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:28.845997+0000 mgr.y (mgr.14144) 303 : cluster [DBG] pgmap v279: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:30.846867+0000 mgr.y (mgr.14144) 304 : cluster [DBG] pgmap v280: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:32.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:30.846867+0000 mgr.y (mgr.14144) 304 : cluster [DBG] pgmap v280: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:32.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:30.846867+0000 mgr.y (mgr.14144) 304 : cluster [DBG] pgmap v280: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:32.847584+0000 mgr.y (mgr.14144) 305 : cluster [DBG] pgmap v281: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:34.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:32.847584+0000 mgr.y (mgr.14144) 305 : cluster [DBG] pgmap v281: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:34.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:32.847584+0000 mgr.y (mgr.14144) 305 : cluster [DBG] pgmap v281: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:36.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:34.848468+0000 mgr.y (mgr.14144) 306 : cluster [DBG] pgmap v282: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:36.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:34.848468+0000 mgr.y (mgr.14144) 306 : cluster [DBG] pgmap v282: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:34.848468+0000 mgr.y (mgr.14144) 306 : cluster [DBG] pgmap v282: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:38.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:36.849197+0000 mgr.y (mgr.14144) 307 : cluster [DBG] pgmap v283: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:36.849197+0000 mgr.y (mgr.14144) 307 : cluster [DBG] pgmap v283: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:36.849197+0000 mgr.y (mgr.14144) 307 : cluster [DBG] pgmap v283: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:38.850155+0000 mgr.y (mgr.14144) 308 : cluster [DBG] pgmap v284: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:40.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:38.850155+0000 mgr.y (mgr.14144) 308 : cluster [DBG] pgmap v284: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:40.402 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:38.850155+0000 mgr.y (mgr.14144) 308 : cluster [DBG] pgmap v284: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:42.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:40.850767+0000 mgr.y (mgr.14144) 309 : cluster [DBG] pgmap v285: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:40.850767+0000 mgr.y (mgr.14144) 309 : cluster [DBG] pgmap v285: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:40.850767+0000 mgr.y (mgr.14144) 309 : cluster [DBG] pgmap v285: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T08:51:44.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:42.851304+0000 mgr.y (mgr.14144) 310 : cluster [DBG] pgmap v286: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:44.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:42.851304+0000 mgr.y (mgr.14144) 310 : cluster [DBG] pgmap v286: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:44.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:42.851304+0000 mgr.y (mgr.14144) 310 : cluster [DBG] pgmap v286: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:44.852153+0000 mgr.y (mgr.14144) 311 : cluster [DBG] pgmap v287: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:44.852153+0000 mgr.y (mgr.14144) 311 : cluster [DBG] pgmap v287: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:46.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:44.852153+0000 mgr.y (mgr.14144) 311 : cluster [DBG] pgmap v287: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:46.852709+0000 mgr.y (mgr.14144) 312 : cluster [DBG] pgmap v288: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:46.852709+0000 mgr.y (mgr.14144) 312 : cluster [DBG] pgmap v288: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:46.852709+0000 mgr.y (mgr.14144) 312 : cluster [DBG] pgmap v288: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:49.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:51:48.831395+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:51:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:51:48.831395+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:51:49.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:51:48.831395+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:51:50.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:48.853384+0000 mgr.y (mgr.14144) 313 : cluster [DBG] pgmap v289: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:50.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:51:49.069438+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:51:50.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:48.853384+0000 mgr.y (mgr.14144) 313 : cluster [DBG] pgmap v289: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:50.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:51:49.069438+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:51:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:48.853384+0000 mgr.y (mgr.14144) 313 : cluster [DBG] pgmap v289: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:50.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:51:49.069438+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:51:52.332 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:50.853969+0000 mgr.y (mgr.14144) 314 : cluster [DBG] pgmap v290: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:52.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:50.853969+0000 mgr.y (mgr.14144) 314 : cluster [DBG] pgmap v290: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:52.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:50.853969+0000 mgr.y (mgr.14144) 314 : cluster [DBG] pgmap v290: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:54.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:52.854664+0000 mgr.y (mgr.14144) 315 : cluster [DBG] pgmap v291: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:52.854664+0000 mgr.y (mgr.14144) 315 : cluster [DBG] pgmap v291: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:54.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:52.854664+0000 mgr.y (mgr.14144) 315 : cluster [DBG] pgmap v291: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:51:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:54.855397+0000 mgr.y (mgr.14144) 316 : cluster [DBG] pgmap v292: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:54.855397+0000 mgr.y (mgr.14144) 316 : cluster [DBG] pgmap v292: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:54.855397+0000 mgr.y (mgr.14144) 316 : cluster [DBG] pgmap v292: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:56.855926+0000 mgr.y (mgr.14144) 317 : cluster [DBG] pgmap v293: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:56.855926+0000 mgr.y (mgr.14144) 317 : cluster [DBG] pgmap v293: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:51:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:56.855926+0000 mgr.y (mgr.14144) 317 : cluster [DBG] pgmap v293: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:51:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:51:58.856765+0000 mgr.y (mgr.14144) 318 : cluster [DBG] pgmap v294: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:51:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:51:58.856765+0000 mgr.y (mgr.14144) 318 : cluster [DBG] pgmap v294: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:00.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:51:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:51:58.856765+0000 mgr.y (mgr.14144) 318 : cluster [DBG] pgmap v294: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:02.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:00.857325+0000 mgr.y (mgr.14144) 319 : cluster [DBG] pgmap v295: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:00.857325+0000 mgr.y (mgr.14144) 319 : cluster [DBG] pgmap v295: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:00.857325+0000 mgr.y (mgr.14144) 319 : cluster [DBG] pgmap v295: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:02.857899+0000 mgr.y (mgr.14144) 320 : cluster [DBG] pgmap v296: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:02.857899+0000 mgr.y (mgr.14144) 320 : cluster [DBG] pgmap v296: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:02.857899+0000 mgr.y (mgr.14144) 320 : cluster [DBG] pgmap v296: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:06.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:04.858521+0000 mgr.y (mgr.14144) 321 : cluster [DBG] pgmap v297: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:04.858521+0000 mgr.y (mgr.14144) 321 : cluster [DBG] pgmap v297: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:06.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:04.858521+0000 mgr.y (mgr.14144) 321 : cluster [DBG] pgmap v297: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:08.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:06.859173+0000 mgr.y (mgr.14144) 322 : cluster [DBG] pgmap v298: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:06.859173+0000 mgr.y (mgr.14144) 322 : cluster [DBG] pgmap v298: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:06.859173+0000 mgr.y (mgr.14144) 322 : cluster [DBG] pgmap v298: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:10.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:08.860096+0000 mgr.y (mgr.14144) 323 : cluster [DBG] pgmap v299: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:08.860096+0000 mgr.y (mgr.14144) 323 : cluster [DBG] pgmap v299: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:08.860096+0000 mgr.y (mgr.14144) 323 : cluster [DBG] pgmap v299: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:11.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:10.904246+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:52:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:10.905738+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:52:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:10.904246+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:52:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:10.905738+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:52:11.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:10.904246+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:52:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:10.905738+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:52:12.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:10.860676+0000 mgr.y (mgr.14144) 324 : cluster [DBG] pgmap v300: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:10.860676+0000 mgr.y (mgr.14144) 324 : cluster [DBG] pgmap v300: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:10.860676+0000 mgr.y (mgr.14144) 324 : cluster [DBG] pgmap v300: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:12.861239+0000 mgr.y (mgr.14144) 325 : cluster [DBG] pgmap v301: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:12.861239+0000 mgr.y (mgr.14144) 325 : cluster [DBG] pgmap v301: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:12.861239+0000 mgr.y (mgr.14144) 325 : cluster [DBG] pgmap v301: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:14.861992+0000 mgr.y (mgr.14144) 326 : cluster [DBG] pgmap v302: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:14.861992+0000 mgr.y (mgr.14144) 326 : cluster [DBG] pgmap v302: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:14.861992+0000 mgr.y (mgr.14144) 326 : cluster [DBG] pgmap v302: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:18.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:16.862725+0000 mgr.y (mgr.14144) 327 : cluster [DBG] pgmap v303: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:16.862725+0000 mgr.y (mgr.14144) 327 : cluster [DBG] pgmap v303: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:16.862725+0000 mgr.y (mgr.14144) 327 : cluster [DBG] pgmap v303: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:18.863462+0000 mgr.y (mgr.14144) 328 : cluster [DBG] pgmap v304: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:18.863462+0000 mgr.y (mgr.14144) 328 : cluster [DBG] pgmap v304: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:20.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:18.863462+0000 mgr.y (mgr.14144) 328 : cluster [DBG] pgmap v304: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:22.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:20.864134+0000 mgr.y (mgr.14144) 329 : cluster [DBG] pgmap v305: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:20.864134+0000 mgr.y (mgr.14144) 329 : cluster [DBG] pgmap v305: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:20.864134+0000 mgr.y (mgr.14144) 329 : cluster [DBG] pgmap v305: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:24.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:22.864720+0000 mgr.y (mgr.14144) 330 : cluster [DBG] pgmap v306: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:22.864720+0000 mgr.y (mgr.14144) 330 : cluster [DBG] pgmap v306: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:24.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:22.864720+0000 mgr.y (mgr.14144) 330 : cluster [DBG] pgmap v306: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:26.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:24.865529+0000 mgr.y (mgr.14144) 331 : cluster [DBG] pgmap v307: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:26.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:24.865529+0000 mgr.y (mgr.14144) 331 : cluster [DBG] pgmap v307: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:26.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:24.865529+0000 mgr.y (mgr.14144) 331 : cluster [DBG] pgmap v307: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:27.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T08:52:28.151 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:26.866049+0000 mgr.y (mgr.14144) 332 : cluster [DBG] pgmap v308: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:28.151 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:26.866049+0000 mgr.y (mgr.14144) 332 : cluster [DBG] pgmap v308: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:26.866049+0000 mgr.y (mgr.14144) 332 : cluster [DBG] pgmap v308: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:28.866823+0000 mgr.y (mgr.14144) 333 : cluster [DBG] pgmap v309: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:29.677273+0000 mon.a (mon.0) 590 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:30.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:28.866823+0000 mgr.y (mgr.14144) 333 : cluster [DBG] pgmap v309: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:30.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:29.677273+0000 mon.a (mon.0) 590 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:28.866823+0000 mgr.y (mgr.14144) 333 : cluster [DBG] pgmap v309: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:29.677273+0000 mon.a (mon.0) 590 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:30.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T08:52:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:30.867346+0000 mgr.y (mgr.14144) 334 : cluster [DBG] pgmap v310: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:30.867346+0000 mgr.y (mgr.14144) 334 : cluster [DBG] pgmap v310: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:30.867346+0000 mgr.y (mgr.14144) 334 : cluster [DBG] pgmap v310: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:32.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T08:52:33.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:32.318827+0000 mon.a (mon.0) 591 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:33.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:32.318827+0000 mon.a (mon.0) 591 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:33.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:32.318827+0000 mon.a (mon.0) 591 : audit [INF] from='client.? ' entity='client.admin' 2024-09-17T08:52:34.535 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:32.867895+0000 mgr.y (mgr.14144) 335 : cluster [DBG] pgmap v311: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:34.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:32.867895+0000 mgr.y (mgr.14144) 335 : cluster [DBG] pgmap v311: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:32.867895+0000 mgr.y (mgr.14144) 335 : cluster [DBG] pgmap v311: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:35.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T08:52:36.054 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:34.868421+0000 mgr.y (mgr.14144) 336 : cluster [DBG] pgmap v312: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:34.895804+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.27:0/1034952385' entity='client.admin' 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:34.899923+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:34.905868+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:34.868421+0000 mgr.y (mgr.14144) 336 : cluster [DBG] pgmap v312: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:34.895804+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.27:0/1034952385' entity='client.admin' 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:34.899923+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:36.055 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:34.905868+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:34.868421+0000 mgr.y (mgr.14144) 336 : cluster [DBG] pgmap v312: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:34.895804+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.27:0/1034952385' entity='client.admin' 2024-09-17T08:52:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:34.899923+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:34.905868+0000 mon.a (mon.0) 594 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:37.460 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-09-17T08:52:37.460 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 running (3m) 2m ago 3m *:9093 *:9094 0.20.0 0881eb8f169f 8b4e9d16b1f5 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 running (3m) 108s ago 3m *:3000 6.7.4 557c83e11646 29288f57deb2 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (3m) 108s ago 3m - 3.4 24ecd6d5f14c cc6bf6c8823e 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 running (7m) 108s ago 7m *:8443 16.2.0 24ecd6d5f14c 2b71602f2019 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 running (9m) 2m ago 9m *:9283 16.2.0 24ecd6d5f14c 564f13fefc6d 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 2m ago 9m - 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (7m) 108s ago 7m - 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (8m) 2m ago 8m - 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 running (4m) 2m ago 4m *:9100 0.18.1 e5a616e4b9cf e705ab22e1ab 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 running (4m) 108s ago 4m *:9100 0.18.1 e5a616e4b9cf 9a64810c80d0 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 2m ago 7m - 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 2m ago 7m - 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T08:52:37.461 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (6m) 2m ago 6m - 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (6m) 2m ago 6m - 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (6m) 108s ago 6m - 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (5m) 108s ago 5m - 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (5m) 108s ago 5m - 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (5m) 108s ago 5m - 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 running (2m) 108s ago 4m *:9095 2.18.1 de242295e225 d7c6a1537970 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 running (3m) 2m ago 3m *:8000 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T08:52:37.462 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 running (3m) 108s ago 3m *:8000 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T08:52:38.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:36.868959+0000 mgr.y (mgr.14144) 337 : cluster [DBG] pgmap v313: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:38.061 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:37.452373+0000 mgr.y (mgr.14144) 338 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:38.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:36.868959+0000 mgr.y (mgr.14144) 337 : cluster [DBG] pgmap v313: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:38.062 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:37.452373+0000 mgr.y (mgr.14144) 338 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:38.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T08:52:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:36.868959+0000 mgr.y (mgr.14144) 337 : cluster [DBG] pgmap v313: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:37.452373+0000 mgr.y (mgr.14144) 338 : audit [DBG] from='client.14739 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:40.033 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T08:52:40.034 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 15 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:52:40.035 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:52:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:38.869748+0000 mgr.y (mgr.14144) 339 : cluster [DBG] pgmap v314: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:38.869748+0000 mgr.y (mgr.14144) 339 : cluster [DBG] pgmap v314: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:38.869748+0000 mgr.y (mgr.14144) 339 : cluster [DBG] pgmap v314: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:40.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph -s' 2024-09-17T08:52:40.955 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:40.033420+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.27:0/2152899513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:52:40.955 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:40.033420+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.27:0/2152899513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:52:41.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:40.033420+0000 mon.a (mon.0) 595 : audit [DBG] from='client.? 172.21.15.27:0/2152899513' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:52:42.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:40.870212+0000 mgr.y (mgr.14144) 340 : cluster [DBG] pgmap v315: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:42.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:40.870212+0000 mgr.y (mgr.14144) 340 : cluster [DBG] pgmap v315: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:40.870212+0000 mgr.y (mgr.14144) 340 : cluster [DBG] pgmap v315: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: id: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 7m) 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 8m), standbys: x 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T08:52:42.597 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 331 objects, 27 KiB 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-17T08:52:42.598 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:52:43.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:42.597025+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.27:0/1627321724' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:52:43.201 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:42.597025+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.27:0/1627321724' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:52:43.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ls' 2024-09-17T08:52:43.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:42.597025+0000 mon.a (mon.0) 596 : audit [DBG] from='client.? 172.21.15.27:0/1627321724' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:52:44.148 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:42.870870+0000 mgr.y (mgr.14144) 341 : cluster [DBG] pgmap v316: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:44.148 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:42.870870+0000 mgr.y (mgr.14144) 341 : cluster [DBG] pgmap v316: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:44.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:42.870870+0000 mgr.y (mgr.14144) 341 : cluster [DBG] pgmap v316: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:45.226 INFO:teuthology.orchestra.run.smithi027.stdout:NAME RUNNING REFRESHED AGE PLACEMENT IMAGE ID 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager 1/1 3m ago 5m smithi027=a;count:1 0881eb8f169f 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:grafana 1/1 116s ago 5m smithi105=a;count:1 557c83e11646 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo 1/1 116s ago 3m count:1 24ecd6d5f14c 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:mgr 2/2 3m ago 7m smithi027=y;smithi105=x;count:2 24ecd6d5f14c 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:mon 3/3 3m ago 8m smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi105:172.21.15.105=b;count:3 24ecd6d5f14c 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter 2/2 3m ago 5m smithi027=a;smithi105=b;count:2 e5a616e4b9cf 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.None 8/0 3m ago - 24ecd6d5f14c 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus 1/1 116s ago 5m smithi105=a;count:1 de242295e225 2024-09-17T08:52:45.227 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo 2/2 3m ago 3m count:2 24ecd6d5f14c 2024-09-17T08:52:45.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T08:52:46.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:44.871400+0000 mgr.y (mgr.14144) 342 : cluster [DBG] pgmap v317: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:45.221563+0000 mgr.y (mgr.14144) 343 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:46.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:44.871400+0000 mgr.y (mgr.14144) 342 : cluster [DBG] pgmap v317: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:46.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:45.221563+0000 mgr.y (mgr.14144) 343 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:44.871400+0000 mgr.y (mgr.14144) 342 : cluster [DBG] pgmap v317: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:45.221563+0000 mgr.y (mgr.14144) 343 : audit [DBG] from='client.14757 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:48.137 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled to redeploy mgr.x on host 'smithi105' 2024-09-17T08:52:48.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:46.871932+0000 mgr.y (mgr.14144) 344 : cluster [DBG] pgmap v318: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:48.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:47.813671+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.27:0/4160791552' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:52:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:46.871932+0000 mgr.y (mgr.14144) 344 : cluster [DBG] pgmap v318: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:47.813671+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.27:0/4160791552' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:52:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:46.871932+0000 mgr.y (mgr.14144) 344 : cluster [DBG] pgmap v318: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:47.813671+0000 mon.a (mon.0) 597 : audit [DBG] from='client.? 172.21.15.27:0/4160791552' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:52:48.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps --refresh' 2024-09-17T08:52:49.417 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:48.130694+0000 mgr.y (mgr.14144) 345 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:48.137085+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:52:48.137627+0000 mgr.y (mgr.14144) 346 : cephadm [INF] Schedule redeploy daemon mgr.x 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:48.137947+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:48.130694+0000 mgr.y (mgr.14144) 345 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:48.137085+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:52:48.137627+0000 mgr.y (mgr.14144) 346 : cephadm [INF] Schedule redeploy daemon mgr.x 2024-09-17T08:52:49.418 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:48.137947+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:49.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:48.130694+0000 mgr.y (mgr.14144) 345 : audit [DBG] from='client.24703 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:48.137085+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:52:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:52:48.137627+0000 mgr.y (mgr.14144) 346 : cephadm [INF] Schedule redeploy daemon mgr.x 2024-09-17T08:52:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:48.137947+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:52:50.444 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:48.872552+0000 mgr.y (mgr.14144) 347 : cluster [DBG] pgmap v319: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:50.444 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:48.872552+0000 mgr.y (mgr.14144) 347 : cluster [DBG] pgmap v319: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:50.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:48.872552+0000 mgr.y (mgr.14144) 347 : cluster [DBG] pgmap v319: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:50.927 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-09-17T08:52:50.927 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 running (3m) 3m ago 4m *:9093 *:9094 0.20.0 0881eb8f169f 8b4e9d16b1f5 2024-09-17T08:52:50.927 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 running (3m) 2m ago 3m *:3000 6.7.4 557c83e11646 29288f57deb2 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (3m) 2m ago 3m - 3.4 24ecd6d5f14c cc6bf6c8823e 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 running (7m) 2m ago 7m *:8443 16.2.0 24ecd6d5f14c 2b71602f2019 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 running (9m) 3m ago 9m *:9283 16.2.0 24ecd6d5f14c 564f13fefc6d 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 3m ago 9m - 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (8m) 2m ago 8m - 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (8m) 3m ago 8m - 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 running (4m) 3m ago 4m *:9100 0.18.1 e5a616e4b9cf e705ab22e1ab 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 running (4m) 2m ago 4m *:9100 0.18.1 e5a616e4b9cf 9a64810c80d0 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 3m ago 7m - 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 3m ago 7m - 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (7m) 3m ago 6m - 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T08:52:50.928 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (6m) 3m ago 6m - 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (6m) 2m ago 6m - 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (6m) 2m ago 6m - 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (5m) 2m ago 5m - 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (5m) 2m ago 5m - 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 running (3m) 2m ago 4m *:9095 2.18.1 de242295e225 d7c6a1537970 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 running (3m) 3m ago 3m *:8000 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T08:52:50.929 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 running (3m) 2m ago 3m *:8000 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T08:52:51.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T08:52:52.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:50.873090+0000 mgr.y (mgr.14144) 348 : cluster [DBG] pgmap v320: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:52.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:52:50.918529+0000 mgr.y (mgr.14144) 349 : audit [DBG] from='client.24709 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:52.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:52:50.918889+0000 mgr.y (mgr.14144) 350 : cephadm [INF] Kicked serve() loop to refresh all daemons 2024-09-17T08:52:52.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:50.873090+0000 mgr.y (mgr.14144) 348 : cluster [DBG] pgmap v320: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:52.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:52:50.918529+0000 mgr.y (mgr.14144) 349 : audit [DBG] from='client.24709 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:52.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:52:50.918889+0000 mgr.y (mgr.14144) 350 : cephadm [INF] Kicked serve() loop to refresh all daemons 2024-09-17T08:52:52.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:50.873090+0000 mgr.y (mgr.14144) 348 : cluster [DBG] pgmap v320: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:52.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:52:50.918529+0000 mgr.y (mgr.14144) 349 : audit [DBG] from='client.24709 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:52:52.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:52:50.918889+0000 mgr.y (mgr.14144) 350 : cephadm [INF] Kicked serve() loop to refresh all daemons 2024-09-17T08:52:53.941 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:52.873655+0000 mgr.y (mgr.14144) 351 : cluster [DBG] pgmap v321: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:53.942 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:52.873655+0000 mgr.y (mgr.14144) 351 : cluster [DBG] pgmap v321: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:52.873655+0000 mgr.y (mgr.14144) 351 : cluster [DBG] pgmap v321: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:52:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:54.874204+0000 mgr.y (mgr.14144) 352 : cluster [DBG] pgmap v322: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:54.874204+0000 mgr.y (mgr.14144) 352 : cluster [DBG] pgmap v322: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:54.874204+0000 mgr.y (mgr.14144) 352 : cluster [DBG] pgmap v322: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:56.874641+0000 mgr.y (mgr.14144) 353 : cluster [DBG] pgmap v323: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:56.874641+0000 mgr.y (mgr.14144) 353 : cluster [DBG] pgmap v323: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:52:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:56.874641+0000 mgr.y (mgr.14144) 353 : cluster [DBG] pgmap v323: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:00.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:52:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:52:58.875234+0000 mgr.y (mgr.14144) 354 : cluster [DBG] pgmap v324: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:52:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:52:58.875234+0000 mgr.y (mgr.14144) 354 : cluster [DBG] pgmap v324: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:52:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:52:58.875234+0000 mgr.y (mgr.14144) 354 : cluster [DBG] pgmap v324: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:02.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:00.875672+0000 mgr.y (mgr.14144) 355 : cluster [DBG] pgmap v325: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:00.875672+0000 mgr.y (mgr.14144) 355 : cluster [DBG] pgmap v325: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:00.875672+0000 mgr.y (mgr.14144) 355 : cluster [DBG] pgmap v325: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:02.876121+0000 mgr.y (mgr.14144) 356 : cluster [DBG] pgmap v326: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:04.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:02.876121+0000 mgr.y (mgr.14144) 356 : cluster [DBG] pgmap v326: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:02.876121+0000 mgr.y (mgr.14144) 356 : cluster [DBG] pgmap v326: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:06.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:04.876631+0000 mgr.y (mgr.14144) 357 : cluster [DBG] pgmap v327: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:04.876631+0000 mgr.y (mgr.14144) 357 : cluster [DBG] pgmap v327: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:04.876631+0000 mgr.y (mgr.14144) 357 : cluster [DBG] pgmap v327: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:08.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:06.877124+0000 mgr.y (mgr.14144) 358 : cluster [DBG] pgmap v328: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:06.877124+0000 mgr.y (mgr.14144) 358 : cluster [DBG] pgmap v328: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:06.877124+0000 mgr.y (mgr.14144) 358 : cluster [DBG] pgmap v328: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:08.877703+0000 mgr.y (mgr.14144) 359 : cluster [DBG] pgmap v329: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:08.877703+0000 mgr.y (mgr.14144) 359 : cluster [DBG] pgmap v329: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:08.877703+0000 mgr.y (mgr.14144) 359 : cluster [DBG] pgmap v329: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:11.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:10.906286+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:53:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:10.908122+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:53:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:10.906286+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:53:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:10.908122+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:53:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:10.906286+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:53:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:10.908122+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:53:12.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:10.878132+0000 mgr.y (mgr.14144) 360 : cluster [DBG] pgmap v330: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:10.878132+0000 mgr.y (mgr.14144) 360 : cluster [DBG] pgmap v330: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:10.878132+0000 mgr.y (mgr.14144) 360 : cluster [DBG] pgmap v330: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:13.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:12.878712+0000 mgr.y (mgr.14144) 361 : cluster [DBG] pgmap v331: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:12.878712+0000 mgr.y (mgr.14144) 361 : cluster [DBG] pgmap v331: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:12.878712+0000 mgr.y (mgr.14144) 361 : cluster [DBG] pgmap v331: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:14.879239+0000 mgr.y (mgr.14144) 362 : cluster [DBG] pgmap v332: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:14.879239+0000 mgr.y (mgr.14144) 362 : cluster [DBG] pgmap v332: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:14.879239+0000 mgr.y (mgr.14144) 362 : cluster [DBG] pgmap v332: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:18.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:16.879849+0000 mgr.y (mgr.14144) 363 : cluster [DBG] pgmap v333: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:16.879849+0000 mgr.y (mgr.14144) 363 : cluster [DBG] pgmap v333: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:16.879849+0000 mgr.y (mgr.14144) 363 : cluster [DBG] pgmap v333: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:18.880587+0000 mgr.y (mgr.14144) 364 : cluster [DBG] pgmap v334: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:18.880587+0000 mgr.y (mgr.14144) 364 : cluster [DBG] pgmap v334: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:20.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:18.880587+0000 mgr.y (mgr.14144) 364 : cluster [DBG] pgmap v334: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:20.881150+0000 mgr.y (mgr.14144) 365 : cluster [DBG] pgmap v335: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:20.881150+0000 mgr.y (mgr.14144) 365 : cluster [DBG] pgmap v335: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:20.881150+0000 mgr.y (mgr.14144) 365 : cluster [DBG] pgmap v335: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:23.570 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:22.499072+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:23.570 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:22.499072+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:23.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:22.499072+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:24.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:22.881724+0000 mgr.y (mgr.14144) 366 : cluster [DBG] pgmap v336: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:24.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:22.881724+0000 mgr.y (mgr.14144) 366 : cluster [DBG] pgmap v336: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:24.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:22.881724+0000 mgr.y (mgr.14144) 366 : cluster [DBG] pgmap v336: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:26.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:24.882397+0000 mgr.y (mgr.14144) 367 : cluster [DBG] pgmap v337: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:25.042644+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:24.882397+0000 mgr.y (mgr.14144) 367 : cluster [DBG] pgmap v337: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:25.042644+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:24.882397+0000 mgr.y (mgr.14144) 367 : cluster [DBG] pgmap v337: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:26.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:25.042644+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:26.342826+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:26.552293+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:26.556522+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:26.557488+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:26.558301+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:53:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:53:26.559100+0000 mgr.y (mgr.14144) 368 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:26.342826+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:26.552293+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:26.556522+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:26.557488+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:26.558301+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:53:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:53:26.559100+0000 mgr.y (mgr.14144) 368 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:53:27.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:26.342826+0000 mon.a (mon.0) 604 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:26.552293+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:53:27.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:26.556522+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:53:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:26.557488+0000 mon.a (mon.0) 607 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:53:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:26.558301+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:53:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:53:26.559100+0000 mgr.y (mgr.14144) 368 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T08:53:29.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:26.882949+0000 mgr.y (mgr.14144) 369 : cluster [DBG] pgmap v338: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:26.882949+0000 mgr.y (mgr.14144) 369 : cluster [DBG] pgmap v338: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:29.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:26.882949+0000 mgr.y (mgr.14144) 369 : cluster [DBG] pgmap v338: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:31.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:28.883821+0000 mgr.y (mgr.14144) 370 : cluster [DBG] pgmap v339: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:28.883821+0000 mgr.y (mgr.14144) 370 : cluster [DBG] pgmap v339: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:28.883821+0000 mgr.y (mgr.14144) 370 : cluster [DBG] pgmap v339: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:33.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:30.884393+0000 mgr.y (mgr.14144) 371 : cluster [DBG] pgmap v340: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:30.884393+0000 mgr.y (mgr.14144) 371 : cluster [DBG] pgmap v340: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:33.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:30.884393+0000 mgr.y (mgr.14144) 371 : cluster [DBG] pgmap v340: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:32.884951+0000 mgr.y (mgr.14144) 372 : cluster [DBG] pgmap v341: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:32.884951+0000 mgr.y (mgr.14144) 372 : cluster [DBG] pgmap v341: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:34.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:32.884951+0000 mgr.y (mgr.14144) 372 : cluster [DBG] pgmap v341: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:36.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:34.885781+0000 mgr.y (mgr.14144) 373 : cluster [DBG] pgmap v342: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:34.885781+0000 mgr.y (mgr.14144) 373 : cluster [DBG] pgmap v342: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:34.885781+0000 mgr.y (mgr.14144) 373 : cluster [DBG] pgmap v342: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:38.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:36.886327+0000 mgr.y (mgr.14144) 374 : cluster [DBG] pgmap v343: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:36.886327+0000 mgr.y (mgr.14144) 374 : cluster [DBG] pgmap v343: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:36.886327+0000 mgr.y (mgr.14144) 374 : cluster [DBG] pgmap v343: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:38.887312+0000 mgr.y (mgr.14144) 375 : cluster [DBG] pgmap v344: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:38.887312+0000 mgr.y (mgr.14144) 375 : cluster [DBG] pgmap v344: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:38.887312+0000 mgr.y (mgr.14144) 375 : cluster [DBG] pgmap v344: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:42.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:40.887888+0000 mgr.y (mgr.14144) 376 : cluster [DBG] pgmap v345: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:40.887888+0000 mgr.y (mgr.14144) 376 : cluster [DBG] pgmap v345: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:40.887888+0000 mgr.y (mgr.14144) 376 : cluster [DBG] pgmap v345: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:42.888345+0000 mgr.y (mgr.14144) 377 : cluster [DBG] pgmap v346: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:43.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:42.888345+0000 mgr.y (mgr.14144) 377 : cluster [DBG] pgmap v346: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:44.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:42.888345+0000 mgr.y (mgr.14144) 377 : cluster [DBG] pgmap v346: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:44.888963+0000 mgr.y (mgr.14144) 378 : cluster [DBG] pgmap v347: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:44.888963+0000 mgr.y (mgr.14144) 378 : cluster [DBG] pgmap v347: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:46.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:44.888963+0000 mgr.y (mgr.14144) 378 : cluster [DBG] pgmap v347: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:46.889694+0000 mgr.y (mgr.14144) 379 : cluster [DBG] pgmap v348: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:46.889694+0000 mgr.y (mgr.14144) 379 : cluster [DBG] pgmap v348: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:46.889694+0000 mgr.y (mgr.14144) 379 : cluster [DBG] pgmap v348: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:50.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:48.890660+0000 mgr.y (mgr.14144) 380 : cluster [DBG] pgmap v349: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:48.890660+0000 mgr.y (mgr.14144) 380 : cluster [DBG] pgmap v349: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:50.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:48.890660+0000 mgr.y (mgr.14144) 380 : cluster [DBG] pgmap v349: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:52.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:50.891172+0000 mgr.y (mgr.14144) 381 : cluster [DBG] pgmap v350: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:50.891172+0000 mgr.y (mgr.14144) 381 : cluster [DBG] pgmap v350: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:52.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:50.891172+0000 mgr.y (mgr.14144) 381 : cluster [DBG] pgmap v350: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:54.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:52.891746+0000 mgr.y (mgr.14144) 382 : cluster [DBG] pgmap v351: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:52.891746+0000 mgr.y (mgr.14144) 382 : cluster [DBG] pgmap v351: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:52.891746+0000 mgr.y (mgr.14144) 382 : cluster [DBG] pgmap v351: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:53:56.216 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:54.892391+0000 mgr.y (mgr.14144) 383 : cluster [DBG] pgmap v352: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:54.892391+0000 mgr.y (mgr.14144) 383 : cluster [DBG] pgmap v352: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:56.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:54.892391+0000 mgr.y (mgr.14144) 383 : cluster [DBG] pgmap v352: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:57.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:57 smithi105 systemd[1]: Stopping Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:53:57.845 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:57 smithi105 podman[40698]: 2024-09-17 08:53:57.591117139 +0000 UTC m=+0.128886129 container died 2b71602f20198de34098b656e06851530963ae21b93ab59e1a57bf35c3aaa5d1 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T08:53:58.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:53:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:56.893167+0000 mgr.y (mgr.14144) 384 : cluster [DBG] pgmap v353: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:58.246 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:57 smithi105 podman[40698]: 2024-09-17 08:53:57.950677373 +0000 UTC m=+0.488446363 container cleanup 2b71602f20198de34098b656e06851530963ae21b93ab59e1a57bf35c3aaa5d1 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20201204, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T08:53:58.246 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:57 smithi105 podman[40698]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x 2024-09-17T08:53:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:53:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:56.893167+0000 mgr.y (mgr.14144) 384 : cluster [DBG] pgmap v353: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:53:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:56.893167+0000 mgr.y (mgr.14144) 384 : cluster [DBG] pgmap v353: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:53:58.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 podman[40712]: 2024-09-17 08:53:58.245939142 +0000 UTC m=+0.649068874 container remove 2b71602f20198de34098b656e06851530963ae21b93ab59e1a57bf35c3aaa5d1 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, RELEASE=HEAD, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20201204, ceph=True, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Dimitri Savineau , org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557) 2024-09-17T08:53:58.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-17T08:53:58.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-17T08:53:58.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 systemd[1]: Stopped Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:53:58.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 systemd[1]: Starting Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:53:58.892 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 podman[40793]: 2024-09-17 08:53:58.535088542 +0000 UTC m=+0.016212220 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T08:53:58.892 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 podman[40793]: 2024-09-17 08:53:58.655308422 +0000 UTC m=+0.136432096 container create e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-19e2a2b) 2024-09-17T08:53:58.892 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 podman[40793]: 2024-09-17 08:53:58.865843261 +0000 UTC m=+0.346966941 container init e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T08:53:58.893 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 podman[40793]: 2024-09-17 08:53:58.868835071 +0000 UTC m=+0.349958746 container start e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T08:53:59.212 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:58.890+0000 7fbcaa7de640 1 -- 172.21.15.105:0/1091368917 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561b605a2d00 con 0x561b60577000 2024-09-17T08:53:59.213 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:58.985+0000 7fbcad047040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T08:53:59.213 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:58 smithi105 bash[40793]: e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 2024-09-17T08:53:59.213 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 systemd[1]: Started Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:53:59.213 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.045+0000 7fbcad047040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T08:53:59.213 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.210+0000 7fbcad047040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T08:53:59.527 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.285+0000 7fbcad047040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T08:53:59.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.439+0000 7fbcad047040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T08:53:59.904 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.581+0000 7fbcad047040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T08:54:00.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.902+0000 7fbcad047040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T08:54:00.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:53:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:53:59.963+0000 7fbcad047040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T08:54:00.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:00.167+0000 7fbcad047040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T08:54:00.169 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:53:58.893955+0000 mgr.y (mgr.14144) 385 : cluster [DBG] pgmap v354: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:00.170 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:59.045297+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:00.170 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:53:59.046639+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:54:00.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:53:58.893955+0000 mgr.y (mgr.14144) 385 : cluster [DBG] pgmap v354: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:59.045297+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:53:59.046639+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:54:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:53:58.893955+0000 mgr.y (mgr.14144) 385 : cluster [DBG] pgmap v354: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:59.045297+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:53:59.046639+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:54:01.380 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:01.077+0000 7fbcad047040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T08:54:01.381 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:01.155+0000 7fbcad047040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T08:54:02.155 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:00.894556+0000 mgr.y (mgr.14144) 386 : cluster [DBG] pgmap v355: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:02.156 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:01.900+0000 7fbcad047040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T08:54:02.156 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.019+0000 7fbcad047040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T08:54:02.446 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.163+0000 7fbcad047040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T08:54:02.446 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.226+0000 7fbcad047040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T08:54:02.446 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.377+0000 7fbcad047040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T08:54:02.447 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.444+0000 7fbcad047040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T08:54:02.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:00.894556+0000 mgr.y (mgr.14144) 386 : cluster [DBG] pgmap v355: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:00.894556+0000 mgr.y (mgr.14144) 386 : cluster [DBG] pgmap v355: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:02.753 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.505+0000 7fbcad047040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T08:54:02.753 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.686+0000 7fbcad047040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T08:54:02.753 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.751+0000 7fbcad047040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T08:54:03.027 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.819+0000 7fbcad047040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T08:54:03.027 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.904+0000 7fbcad047040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T08:54:03.027 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:02.966+0000 7fbcad047040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T08:54:03.027 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:03.025+0000 7fbcad047040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T08:54:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:02.895089+0000 mgr.y (mgr.14144) 387 : cluster [DBG] pgmap v356: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:02.895089+0000 mgr.y (mgr.14144) 387 : cluster [DBG] pgmap v356: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:02.895089+0000 mgr.y (mgr.14144) 387 : cluster [DBG] pgmap v356: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:04.027 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:54:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:54:03.843+0000 7fbcad047040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:03.847143+0000 mon.a (mon.0) 611 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:03.847246+0000 mon.a (mon.0) 612 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:03.847829+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:03.848219+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:03.849120+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:03.849374+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:04.138386+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:04.143077+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:03.847143+0000 mon.a (mon.0) 611 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:03.847246+0000 mon.a (mon.0) 612 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:03.847829+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:03.848219+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:03.849120+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:03.849374+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:04.138386+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:04.143077+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:03.847143+0000 mon.a (mon.0) 611 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:03.847246+0000 mon.a (mon.0) 612 : cluster [DBG] Standby manager daemon x started 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:03.847829+0000 mon.b (mon.2) 24 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:03.848219+0000 mon.b (mon.2) 25 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:03.849120+0000 mon.b (mon.2) 26 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:03.849374+0000 mon.b (mon.2) 27 : audit [DBG] from='mgr.? 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:04.138386+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:04.143077+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:54:06.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:04.681696+0000 mon.a (mon.0) 615 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-17T08:54:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:04.895778+0000 mgr.y (mgr.14144) 388 : cluster [DBG] pgmap v357: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:04.681696+0000 mon.a (mon.0) 615 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-17T08:54:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:04.895778+0000 mgr.y (mgr.14144) 388 : cluster [DBG] pgmap v357: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:06.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:04.681696+0000 mon.a (mon.0) 615 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-17T08:54:06.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:04.895778+0000 mgr.y (mgr.14144) 388 : cluster [DBG] pgmap v357: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:06.896334+0000 mgr.y (mgr.14144) 389 : cluster [DBG] pgmap v358: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:06.896334+0000 mgr.y (mgr.14144) 389 : cluster [DBG] pgmap v358: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:06.896334+0000 mgr.y (mgr.14144) 389 : cluster [DBG] pgmap v358: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:10.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:08.897068+0000 mgr.y (mgr.14144) 390 : cluster [DBG] pgmap v359: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:08.897068+0000 mgr.y (mgr.14144) 390 : cluster [DBG] pgmap v359: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:08.897068+0000 mgr.y (mgr.14144) 390 : cluster [DBG] pgmap v359: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:10.910252+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:54:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:54:10.910521+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:54:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:10.910252+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:54:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:54:10.910521+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:54:11.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:10.910252+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:54:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:54:10.910521+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:54:12.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:10.897579+0000 mgr.y (mgr.14144) 391 : cluster [DBG] pgmap v360: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:10.897579+0000 mgr.y (mgr.14144) 391 : cluster [DBG] pgmap v360: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:10.897579+0000 mgr.y (mgr.14144) 391 : cluster [DBG] pgmap v360: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:12.898127+0000 mgr.y (mgr.14144) 392 : cluster [DBG] pgmap v361: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:12.898127+0000 mgr.y (mgr.14144) 392 : cluster [DBG] pgmap v361: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:14.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:12.898127+0000 mgr.y (mgr.14144) 392 : cluster [DBG] pgmap v361: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:14.898888+0000 mgr.y (mgr.14144) 393 : cluster [DBG] pgmap v362: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:14.898888+0000 mgr.y (mgr.14144) 393 : cluster [DBG] pgmap v362: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:14.898888+0000 mgr.y (mgr.14144) 393 : cluster [DBG] pgmap v362: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:16.899358+0000 mgr.y (mgr.14144) 394 : cluster [DBG] pgmap v363: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:16.899358+0000 mgr.y (mgr.14144) 394 : cluster [DBG] pgmap v363: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:18.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:16.899358+0000 mgr.y (mgr.14144) 394 : cluster [DBG] pgmap v363: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:18.900172+0000 mgr.y (mgr.14144) 395 : cluster [DBG] pgmap v364: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:18.900172+0000 mgr.y (mgr.14144) 395 : cluster [DBG] pgmap v364: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:20.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:18.900172+0000 mgr.y (mgr.14144) 395 : cluster [DBG] pgmap v364: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:20.900870+0000 mgr.y (mgr.14144) 396 : cluster [DBG] pgmap v365: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:20.900870+0000 mgr.y (mgr.14144) 396 : cluster [DBG] pgmap v365: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:20.900870+0000 mgr.y (mgr.14144) 396 : cluster [DBG] pgmap v365: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:24.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:22.901555+0000 mgr.y (mgr.14144) 397 : cluster [DBG] pgmap v366: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:22.901555+0000 mgr.y (mgr.14144) 397 : cluster [DBG] pgmap v366: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:24.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:22.901555+0000 mgr.y (mgr.14144) 397 : cluster [DBG] pgmap v366: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:24.902218+0000 mgr.y (mgr.14144) 398 : cluster [DBG] pgmap v367: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:24.902218+0000 mgr.y (mgr.14144) 398 : cluster [DBG] pgmap v367: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:26.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:24.902218+0000 mgr.y (mgr.14144) 398 : cluster [DBG] pgmap v367: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:28.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:26.902759+0000 mgr.y (mgr.14144) 399 : cluster [DBG] pgmap v368: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:26.902759+0000 mgr.y (mgr.14144) 399 : cluster [DBG] pgmap v368: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:26.902759+0000 mgr.y (mgr.14144) 399 : cluster [DBG] pgmap v368: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:28.903519+0000 mgr.y (mgr.14144) 400 : cluster [DBG] pgmap v369: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:28.903519+0000 mgr.y (mgr.14144) 400 : cluster [DBG] pgmap v369: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:28.903519+0000 mgr.y (mgr.14144) 400 : cluster [DBG] pgmap v369: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:30.904014+0000 mgr.y (mgr.14144) 401 : cluster [DBG] pgmap v370: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:30.904014+0000 mgr.y (mgr.14144) 401 : cluster [DBG] pgmap v370: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:30.904014+0000 mgr.y (mgr.14144) 401 : cluster [DBG] pgmap v370: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:32.904749+0000 mgr.y (mgr.14144) 402 : cluster [DBG] pgmap v371: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:32.904749+0000 mgr.y (mgr.14144) 402 : cluster [DBG] pgmap v371: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:34.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:32.904749+0000 mgr.y (mgr.14144) 402 : cluster [DBG] pgmap v371: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:36.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:34.905519+0000 mgr.y (mgr.14144) 403 : cluster [DBG] pgmap v372: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:34.905519+0000 mgr.y (mgr.14144) 403 : cluster [DBG] pgmap v372: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:34.905519+0000 mgr.y (mgr.14144) 403 : cluster [DBG] pgmap v372: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:38.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:36.906040+0000 mgr.y (mgr.14144) 404 : cluster [DBG] pgmap v373: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:36.906040+0000 mgr.y (mgr.14144) 404 : cluster [DBG] pgmap v373: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:36.906040+0000 mgr.y (mgr.14144) 404 : cluster [DBG] pgmap v373: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:40.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:38.906946+0000 mgr.y (mgr.14144) 405 : cluster [DBG] pgmap v374: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:38.906946+0000 mgr.y (mgr.14144) 405 : cluster [DBG] pgmap v374: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:38.906946+0000 mgr.y (mgr.14144) 405 : cluster [DBG] pgmap v374: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:42.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:40.907620+0000 mgr.y (mgr.14144) 406 : cluster [DBG] pgmap v375: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:40.907620+0000 mgr.y (mgr.14144) 406 : cluster [DBG] pgmap v375: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:42.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:40.907620+0000 mgr.y (mgr.14144) 406 : cluster [DBG] pgmap v375: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:44.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:42.908239+0000 mgr.y (mgr.14144) 407 : cluster [DBG] pgmap v376: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:44.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:42.908239+0000 mgr.y (mgr.14144) 407 : cluster [DBG] pgmap v376: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:42.908239+0000 mgr.y (mgr.14144) 407 : cluster [DBG] pgmap v376: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:44.908943+0000 mgr.y (mgr.14144) 408 : cluster [DBG] pgmap v377: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:44.908943+0000 mgr.y (mgr.14144) 408 : cluster [DBG] pgmap v377: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:44.908943+0000 mgr.y (mgr.14144) 408 : cluster [DBG] pgmap v377: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:46.909441+0000 mgr.y (mgr.14144) 409 : cluster [DBG] pgmap v378: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:46.909441+0000 mgr.y (mgr.14144) 409 : cluster [DBG] pgmap v378: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:46.909441+0000 mgr.y (mgr.14144) 409 : cluster [DBG] pgmap v378: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:50.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:48.910099+0000 mgr.y (mgr.14144) 410 : cluster [DBG] pgmap v379: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:48.910099+0000 mgr.y (mgr.14144) 410 : cluster [DBG] pgmap v379: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:50.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:48.910099+0000 mgr.y (mgr.14144) 410 : cluster [DBG] pgmap v379: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:52.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:50.910784+0000 mgr.y (mgr.14144) 411 : cluster [DBG] pgmap v380: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:50.910784+0000 mgr.y (mgr.14144) 411 : cluster [DBG] pgmap v380: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:50.910784+0000 mgr.y (mgr.14144) 411 : cluster [DBG] pgmap v380: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:54.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:52.911526+0000 mgr.y (mgr.14144) 412 : cluster [DBG] pgmap v381: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:52.911526+0000 mgr.y (mgr.14144) 412 : cluster [DBG] pgmap v381: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:54.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:52.911526+0000 mgr.y (mgr.14144) 412 : cluster [DBG] pgmap v381: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:54:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:54.912143+0000 mgr.y (mgr.14144) 413 : cluster [DBG] pgmap v382: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:54.912143+0000 mgr.y (mgr.14144) 413 : cluster [DBG] pgmap v382: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:54.912143+0000 mgr.y (mgr.14144) 413 : cluster [DBG] pgmap v382: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:56.912670+0000 mgr.y (mgr.14144) 414 : cluster [DBG] pgmap v383: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:56.912670+0000 mgr.y (mgr.14144) 414 : cluster [DBG] pgmap v383: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:54:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:56.912670+0000 mgr.y (mgr.14144) 414 : cluster [DBG] pgmap v383: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:00.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:54:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:54:58.913621+0000 mgr.y (mgr.14144) 415 : cluster [DBG] pgmap v384: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:00.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:54:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:54:58.913621+0000 mgr.y (mgr.14144) 415 : cluster [DBG] pgmap v384: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:00.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:54:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:54:58.913621+0000 mgr.y (mgr.14144) 415 : cluster [DBG] pgmap v384: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:00.914155+0000 mgr.y (mgr.14144) 416 : cluster [DBG] pgmap v385: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:00.914155+0000 mgr.y (mgr.14144) 416 : cluster [DBG] pgmap v385: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:00.914155+0000 mgr.y (mgr.14144) 416 : cluster [DBG] pgmap v385: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:02.914732+0000 mgr.y (mgr.14144) 417 : cluster [DBG] pgmap v386: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:02.914732+0000 mgr.y (mgr.14144) 417 : cluster [DBG] pgmap v386: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:04.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:02.914732+0000 mgr.y (mgr.14144) 417 : cluster [DBG] pgmap v386: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:04.147648+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:04.360084+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:04.374324+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:04.593267+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:04.147648+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:55:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:04.360084+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:04.374324+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:04.593267+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:04.147648+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:55:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:04.360084+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:04.374324+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:04.593267+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:55:06.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:04.915523+0000 mgr.y (mgr.14144) 418 : cluster [DBG] pgmap v387: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:04.915523+0000 mgr.y (mgr.14144) 418 : cluster [DBG] pgmap v387: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:06.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:04.915523+0000 mgr.y (mgr.14144) 418 : cluster [DBG] pgmap v387: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:08.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:06.916043+0000 mgr.y (mgr.14144) 419 : cluster [DBG] pgmap v388: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:06.916043+0000 mgr.y (mgr.14144) 419 : cluster [DBG] pgmap v388: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:06.916043+0000 mgr.y (mgr.14144) 419 : cluster [DBG] pgmap v388: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:10.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:08.916841+0000 mgr.y (mgr.14144) 420 : cluster [DBG] pgmap v389: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:08.916841+0000 mgr.y (mgr.14144) 420 : cluster [DBG] pgmap v389: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:08.916841+0000 mgr.y (mgr.14144) 420 : cluster [DBG] pgmap v389: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:11.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:10.915353+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:55:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:10.915631+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:55:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:10.915353+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:55:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:10.915631+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:55:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:10.915353+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:55:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:10.915631+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:55:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:10.917365+0000 mgr.y (mgr.14144) 421 : cluster [DBG] pgmap v390: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:10.917365+0000 mgr.y (mgr.14144) 421 : cluster [DBG] pgmap v390: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:10.917365+0000 mgr.y (mgr.14144) 421 : cluster [DBG] pgmap v390: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:14.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:12.917941+0000 mgr.y (mgr.14144) 422 : cluster [DBG] pgmap v391: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:12.917941+0000 mgr.y (mgr.14144) 422 : cluster [DBG] pgmap v391: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:12.917941+0000 mgr.y (mgr.14144) 422 : cluster [DBG] pgmap v391: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:14.918575+0000 mgr.y (mgr.14144) 423 : cluster [DBG] pgmap v392: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:14.918575+0000 mgr.y (mgr.14144) 423 : cluster [DBG] pgmap v392: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:14.918575+0000 mgr.y (mgr.14144) 423 : cluster [DBG] pgmap v392: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:18.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:16.919119+0000 mgr.y (mgr.14144) 424 : cluster [DBG] pgmap v393: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:16.919119+0000 mgr.y (mgr.14144) 424 : cluster [DBG] pgmap v393: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:16.919119+0000 mgr.y (mgr.14144) 424 : cluster [DBG] pgmap v393: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T08:55:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:18.919901+0000 mgr.y (mgr.14144) 425 : cluster [DBG] pgmap v394: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:18.919901+0000 mgr.y (mgr.14144) 425 : cluster [DBG] pgmap v394: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:18.919901+0000 mgr.y (mgr.14144) 425 : cluster [DBG] pgmap v394: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:20.920394+0000 mgr.y (mgr.14144) 426 : cluster [DBG] pgmap v395: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:20.920394+0000 mgr.y (mgr.14144) 426 : cluster [DBG] pgmap v395: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:20.920394+0000 mgr.y (mgr.14144) 426 : cluster [DBG] pgmap v395: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:24.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:22.921084+0000 mgr.y (mgr.14144) 427 : cluster [DBG] pgmap v396: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:22.921084+0000 mgr.y (mgr.14144) 427 : cluster [DBG] pgmap v396: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:22.921084+0000 mgr.y (mgr.14144) 427 : cluster [DBG] pgmap v396: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:26.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:24.921780+0000 mgr.y (mgr.14144) 428 : cluster [DBG] pgmap v397: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:24.921780+0000 mgr.y (mgr.14144) 428 : cluster [DBG] pgmap v397: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:24.921780+0000 mgr.y (mgr.14144) 428 : cluster [DBG] pgmap v397: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:28.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:26.922260+0000 mgr.y (mgr.14144) 429 : cluster [DBG] pgmap v398: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:26.922260+0000 mgr.y (mgr.14144) 429 : cluster [DBG] pgmap v398: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:26.922260+0000 mgr.y (mgr.14144) 429 : cluster [DBG] pgmap v398: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:28.923055+0000 mgr.y (mgr.14144) 430 : cluster [DBG] pgmap v399: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:28.923055+0000 mgr.y (mgr.14144) 430 : cluster [DBG] pgmap v399: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:28.923055+0000 mgr.y (mgr.14144) 430 : cluster [DBG] pgmap v399: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:30.923694+0000 mgr.y (mgr.14144) 431 : cluster [DBG] pgmap v400: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:30.923694+0000 mgr.y (mgr.14144) 431 : cluster [DBG] pgmap v400: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:30.923694+0000 mgr.y (mgr.14144) 431 : cluster [DBG] pgmap v400: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:32.924355+0000 mgr.y (mgr.14144) 432 : cluster [DBG] pgmap v401: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:32.924355+0000 mgr.y (mgr.14144) 432 : cluster [DBG] pgmap v401: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:32.924355+0000 mgr.y (mgr.14144) 432 : cluster [DBG] pgmap v401: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:36.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:34.925115+0000 mgr.y (mgr.14144) 433 : cluster [DBG] pgmap v402: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:34.925115+0000 mgr.y (mgr.14144) 433 : cluster [DBG] pgmap v402: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:34.925115+0000 mgr.y (mgr.14144) 433 : cluster [DBG] pgmap v402: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:36.925633+0000 mgr.y (mgr.14144) 434 : cluster [DBG] pgmap v403: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:36.925633+0000 mgr.y (mgr.14144) 434 : cluster [DBG] pgmap v403: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:38.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:36.925633+0000 mgr.y (mgr.14144) 434 : cluster [DBG] pgmap v403: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:38.926350+0000 mgr.y (mgr.14144) 435 : cluster [DBG] pgmap v404: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:38.926350+0000 mgr.y (mgr.14144) 435 : cluster [DBG] pgmap v404: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:40.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:38.926350+0000 mgr.y (mgr.14144) 435 : cluster [DBG] pgmap v404: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:42.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:40.927077+0000 mgr.y (mgr.14144) 436 : cluster [DBG] pgmap v405: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:42.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:40.927077+0000 mgr.y (mgr.14144) 436 : cluster [DBG] pgmap v405: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:40.927077+0000 mgr.y (mgr.14144) 436 : cluster [DBG] pgmap v405: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:44.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:42.927671+0000 mgr.y (mgr.14144) 437 : cluster [DBG] pgmap v406: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:44.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:42.927671+0000 mgr.y (mgr.14144) 437 : cluster [DBG] pgmap v406: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:42.927671+0000 mgr.y (mgr.14144) 437 : cluster [DBG] pgmap v406: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:44.928419+0000 mgr.y (mgr.14144) 438 : cluster [DBG] pgmap v407: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:44.928419+0000 mgr.y (mgr.14144) 438 : cluster [DBG] pgmap v407: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:44.928419+0000 mgr.y (mgr.14144) 438 : cluster [DBG] pgmap v407: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:46.928983+0000 mgr.y (mgr.14144) 439 : cluster [DBG] pgmap v408: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:46.928983+0000 mgr.y (mgr.14144) 439 : cluster [DBG] pgmap v408: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:46.928983+0000 mgr.y (mgr.14144) 439 : cluster [DBG] pgmap v408: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:50.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:48.929785+0000 mgr.y (mgr.14144) 440 : cluster [DBG] pgmap v409: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:48.929785+0000 mgr.y (mgr.14144) 440 : cluster [DBG] pgmap v409: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:50.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:48.929785+0000 mgr.y (mgr.14144) 440 : cluster [DBG] pgmap v409: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:50.930296+0000 mgr.y (mgr.14144) 441 : cluster [DBG] pgmap v410: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:52.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:50.930296+0000 mgr.y (mgr.14144) 441 : cluster [DBG] pgmap v410: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:52.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:50.930296+0000 mgr.y (mgr.14144) 441 : cluster [DBG] pgmap v410: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:54.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:52.930971+0000 mgr.y (mgr.14144) 442 : cluster [DBG] pgmap v411: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:52.930971+0000 mgr.y (mgr.14144) 442 : cluster [DBG] pgmap v411: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:52.930971+0000 mgr.y (mgr.14144) 442 : cluster [DBG] pgmap v411: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:55:55.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T08:55:56.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:54.931580+0000 mgr.y (mgr.14144) 443 : cluster [DBG] pgmap v412: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:54.931580+0000 mgr.y (mgr.14144) 443 : cluster [DBG] pgmap v412: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:56.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:54.931580+0000 mgr.y (mgr.14144) 443 : cluster [DBG] pgmap v412: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:57.175 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST STATUS REFRESHED AGE PORTS VERSION IMAGE ID CONTAINER ID 2024-09-17T08:55:57.175 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 running (6m) 2m ago 7m *:9093 *:9094 0.20.0 0881eb8f169f 8b4e9d16b1f5 2024-09-17T08:55:57.175 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 running (6m) 113s ago 6m *:3000 6.7.4 557c83e11646 29288f57deb2 2024-09-17T08:55:57.176 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (6m) 113s ago 6m - 3.4 24ecd6d5f14c cc6bf6c8823e 2024-09-17T08:55:57.176 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 running (118s) 113s ago 11m *:8443 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T08:55:57.176 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 running (12m) 2m ago 12m *:9283 16.2.0 24ecd6d5f14c 564f13fefc6d 2024-09-17T08:55:57.176 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (12m) 2m ago 12m - 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T08:55:57.176 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (11m) 113s ago 11m - 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T08:55:57.177 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (11m) 2m ago 11m - 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T08:55:57.177 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 running (7m) 2m ago 7m *:9100 0.18.1 e5a616e4b9cf e705ab22e1ab 2024-09-17T08:55:57.177 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 running (7m) 113s ago 7m *:9100 0.18.1 e5a616e4b9cf 9a64810c80d0 2024-09-17T08:55:57.177 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (10m) 2m ago 10m - 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T08:55:57.177 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (10m) 2m ago 10m - 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (10m) 2m ago 10m - 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (9m) 2m ago 9m - 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (9m) 113s ago 9m - 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (9m) 113s ago 9m - 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (8m) 113s ago 8m - 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T08:55:57.178 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (8m) 113s ago 8m - 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T08:55:57.179 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 running (6m) 113s ago 8m *:9095 2.18.1 de242295e225 d7c6a1537970 2024-09-17T08:55:57.179 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 running (6m) 2m ago 6m *:8000 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T08:55:57.179 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 running (6m) 113s ago 6m *:8000 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T08:55:57.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T08:55:58.141 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:56.932028+0000 mgr.y (mgr.14144) 444 : cluster [DBG] pgmap v413: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:58.142 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:57.167759+0000 mgr.y (mgr.14144) 445 : audit [DBG] from='client.24718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:55:58.142 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:56.932028+0000 mgr.y (mgr.14144) 444 : cluster [DBG] pgmap v413: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:58.142 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:57.167759+0000 mgr.y (mgr.14144) 445 : audit [DBG] from='client.24718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:55:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:56.932028+0000 mgr.y (mgr.14144) 444 : cluster [DBG] pgmap v413: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:55:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:57.167759+0000 mgr.y (mgr.14144) 445 : audit [DBG] from='client.24718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T08:55:59.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 14, 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T08:55:59.871 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T08:56:00.181 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:55:58.932703+0000 mgr.y (mgr.14144) 446 : cluster [DBG] pgmap v414: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:00.181 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:55:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:55:59.869052+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.27:0/1640704170' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:00.182 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:55:58.932703+0000 mgr.y (mgr.14144) 446 : cluster [DBG] pgmap v414: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:00.182 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:55:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:55:59.869052+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.27:0/1640704170' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:55:58.932703+0000 mgr.y (mgr.14144) 446 : cluster [DBG] pgmap v414: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:55:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:55:59.869052+0000 mon.a (mon.0) 624 : audit [DBG] from='client.? 172.21.15.27:0/1640704170' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:00.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph -s' 2024-09-17T08:56:02.074 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:00.933163+0000 mgr.y (mgr.14144) 447 : cluster [DBG] pgmap v415: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:02.075 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:00.933163+0000 mgr.y (mgr.14144) 447 : cluster [DBG] pgmap v415: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:00.933163+0000 mgr.y (mgr.14144) 447 : cluster [DBG] pgmap v415: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:02.344 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-09-17T08:56:02.344 INFO:teuthology.orchestra.run.smithi027.stdout: id: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_OK 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 11m) 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 11m), standbys: x 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 8m) 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 331 objects, 27 KiB 2024-09-17T08:56:02.345 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:02.346 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-09-17T08:56:02.346 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:56:02.346 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-09-17T08:56:02.346 INFO:teuthology.orchestra.run.smithi027.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-09-17T08:56:02.346 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T08:56:02.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T08:56:03.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:02.344423+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.27:0/2472721071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:56:03.256 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:02.344423+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.27:0/2472721071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:56:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:02.344423+0000 mon.c (mon.1) 21 : audit [DBG] from='client.? 172.21.15.27:0/2472721071' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T08:56:04.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:02.933745+0000 mgr.y (mgr.14144) 448 : cluster [DBG] pgmap v416: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:02.933745+0000 mgr.y (mgr.14144) 448 : cluster [DBG] pgmap v416: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:02.933745+0000 mgr.y (mgr.14144) 448 : cluster [DBG] pgmap v416: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:05.117 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-09-17T08:56:05.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:04.597268+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:04.839588+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:56:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:04.597268+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:04.839588+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:56:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:04.597268+0000 mon.a (mon.0) 625 : audit [DBG] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:04.839588+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' 2024-09-17T08:56:05.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-09-17T08:56:06.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:04.934238+0000 mgr.y (mgr.14144) 449 : cluster [DBG] pgmap v417: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:05.116477+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.27:0/28817678' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T08:56:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:04.934238+0000 mgr.y (mgr.14144) 449 : cluster [DBG] pgmap v417: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:05.116477+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.27:0/28817678' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T08:56:06.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:04.934238+0000 mgr.y (mgr.14144) 449 : cluster [DBG] pgmap v417: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:06.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:05.116477+0000 mon.c (mon.1) 22 : audit [DBG] from='client.? 172.21.15.27:0/28817678' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T08:56:07.446 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T08:56:08.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph mgr fail' 2024-09-17T08:56:08.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:06.934799+0000 mgr.y (mgr.14144) 450 : cluster [DBG] pgmap v418: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:07.434651+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.27:0/1992054503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:06.934799+0000 mgr.y (mgr.14144) 450 : cluster [DBG] pgmap v418: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:07.434651+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.27:0/1992054503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:06.934799+0000 mgr.y (mgr.14144) 450 : cluster [DBG] pgmap v418: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:07.434651+0000 mon.a (mon.0) 627 : audit [DBG] from='client.? 172.21.15.27:0/1992054503' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T08:56:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:08.935379+0000 mgr.y (mgr.14144) 451 : cluster [DBG] pgmap v419: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:10.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:08.935379+0000 mgr.y (mgr.14144) 451 : cluster [DBG] pgmap v419: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:10.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:08.935379+0000 mgr.y (mgr.14144) 451 : cluster [DBG] pgmap v419: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:10.061180+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.27:0/1578981995' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:10.061438+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:10.066042+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:10.919988+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:10.920318+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:10.937204+0000 mon.a (mon.0) 632 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:11.404 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:10.061180+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.27:0/1578981995' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:10.061438+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:10.066042+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:10.919988+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:10.920318+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:10.937204+0000 mon.a (mon.0) 632 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:10.061180+0000 mon.c (mon.1) 23 : audit [INF] from='client.? 172.21.15.27:0/1578981995' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:10.061438+0000 mon.a (mon.0) 628 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:10.066042+0000 mon.a (mon.0) 629 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:10.919988+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:10.920318+0000 mon.a (mon.0) 631 : audit [INF] from='mgr.14144 172.21.15.27:0/2216581924' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:10.937204+0000 mon.a (mon.0) 632 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: debug 2024-09-17T08:56:11.021+0000 7f9136151700 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setuser ceph since I am not root 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: ignoring --setgroup ceph since I am not root 2024-09-17T08:56:11.406 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:56:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y[23819]: debug 2024-09-17T08:56:11.073+0000 7fbda2f6c700 1 -- 172.21.15.27:0/2681759074 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dff37ac4e0 con 0x55dff3782c00 2024-09-17T08:56:11.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.025254+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:11.025327+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e17: x(active, starting, since 0.96328s), standbys: y 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.025726+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.025951+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.026171+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.027047+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.027309+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.027566+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.027876+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:56:12.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.028148+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.028434+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.028700+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.028991+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.029264+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.029534+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.025254+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:11.025327+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e17: x(active, starting, since 0.96328s), standbys: y 2024-09-17T08:56:12.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.025726+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.025951+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.026171+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.027047+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.027309+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.027566+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.027876+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.028148+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.028434+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.028700+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.028991+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.029264+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:56:12.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.029534+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.029967+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.030187+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.030679+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:11.192714+0000 mon.a (mon.0) 635 : cluster [INF] Manager daemon x is now available 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:11.203848+0000 mgr.x (mgr.34616) 1 : cephadm [INF] inventory: adjusted host smithi027 addr 'smithi027' -> '172.21.15.27' 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.207626+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.218555+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:11.218960+0000 mgr.x (mgr.34616) 2 : cephadm [INF] Queued rgw.foo for migration 2024-09-17T08:56:12.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.222582+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.248241+0000 mon.b (mon.2) 45 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.248684+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.256785+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.259119+0000 mon.b (mon.2) 46 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.259422+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:11.688487+0000 mgr.x (mgr.34616) 3 : cephadm [INF] Deploying cephadm binary to smithi105 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:11.712248+0000 mgr.x (mgr.34616) 4 : cephadm [INF] Deploying cephadm binary to smithi027 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.754359+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.173 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:11.758486+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.029967+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.030187+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.030679+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:11.192714+0000 mon.a (mon.0) 635 : cluster [INF] Manager daemon x is now available 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:11.203848+0000 mgr.x (mgr.34616) 1 : cephadm [INF] inventory: adjusted host smithi027 addr 'smithi027' -> '172.21.15.27' 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.207626+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.218555+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:11.218960+0000 mgr.x (mgr.34616) 2 : cephadm [INF] Queued rgw.foo for migration 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.222582+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.248241+0000 mon.b (mon.2) 45 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.174 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.248684+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.256785+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.259119+0000 mon.b (mon.2) 46 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.259422+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:11.688487+0000 mgr.x (mgr.34616) 3 : cephadm [INF] Deploying cephadm binary to smithi105 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:11.712248+0000 mgr.x (mgr.34616) 4 : cephadm [INF] Deploying cephadm binary to smithi027 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.754359+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:11.758486+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.025254+0000 mon.a (mon.0) 633 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:11.025327+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e17: x(active, starting, since 0.96328s), standbys: y 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.025726+0000 mon.b (mon.2) 28 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.025951+0000 mon.b (mon.2) 29 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.026171+0000 mon.b (mon.2) 30 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.027047+0000 mon.b (mon.2) 31 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.027309+0000 mon.b (mon.2) 32 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T08:56:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.027566+0000 mon.b (mon.2) 33 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.027876+0000 mon.b (mon.2) 34 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.028148+0000 mon.b (mon.2) 35 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.028434+0000 mon.b (mon.2) 36 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.028700+0000 mon.b (mon.2) 37 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.028991+0000 mon.b (mon.2) 38 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.029264+0000 mon.b (mon.2) 39 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.029534+0000 mon.b (mon.2) 40 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T08:56:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.029967+0000 mon.b (mon.2) 41 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.030187+0000 mon.b (mon.2) 42 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.030679+0000 mon.b (mon.2) 43 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:11.192714+0000 mon.a (mon.0) 635 : cluster [INF] Manager daemon x is now available 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:11.203848+0000 mgr.x (mgr.34616) 1 : cephadm [INF] inventory: adjusted host smithi027 addr 'smithi027' -> '172.21.15.27' 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.207626+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.218555+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:11.218960+0000 mgr.x (mgr.34616) 2 : cephadm [INF] Queued rgw.foo for migration 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.222582+0000 mon.b (mon.2) 44 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.248241+0000 mon.b (mon.2) 45 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.248684+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T08:56:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.256785+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.259119+0000 mon.b (mon.2) 46 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.259422+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:11.688487+0000 mgr.x (mgr.34616) 3 : cephadm [INF] Deploying cephadm binary to smithi105 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:11.712248+0000 mgr.x (mgr.34616) 4 : cephadm [INF] Deploying cephadm binary to smithi027 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.754359+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:12.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:11.758486+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:13.077 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T08:56:12.777+0000 7fbc12c55640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:08:56:12] ENGINE Error in HTTPServer.serve 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: Traceback (most recent call last): 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._connections.run(self.expiration_interval) 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._run(expiration_interval) 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T08:56:13.078 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: s = self.context.wrap_socket( 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: return self.sslsocket_class._create( 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self.do_handshake() 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._sslobj.do_handshake() 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T08:56:13.079 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 08:56:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:12.077992+0000 mon.a (mon.0) 643 : cluster [DBG] mgrmap e18: x(active, since 2s), standbys: y 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:12.087249+0000 mgr.x (mgr.34616) 5 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:12.659749+0000 mgr.x (mgr.34616) 6 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTING 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:12.775505+0000 mgr.x (mgr.34616) 7 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:12.777979+0000 mgr.x (mgr.34616) 8 : cephadm [ERR] [17/Sep/2024:08:56:12] ENGINE Error in HTTPServer.serve 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Traceback (most recent call last): 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._connections.run(self.expiration_interval) 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._run(expiration_interval) 2024-09-17T08:56:13.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s = self.context.wrap_socket( 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: return self.sslsocket_class._create( 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self.do_handshake() 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._sslobj.do_handshake() 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T08:56:13.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:12.876487+0000 mgr.x (mgr.34616) 9 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T08:56:13.082 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:12.876724+0000 mgr.x (mgr.34616) 10 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTED 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:12.077992+0000 mon.a (mon.0) 643 : cluster [DBG] mgrmap e18: x(active, since 2s), standbys: y 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:12.087249+0000 mgr.x (mgr.34616) 5 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:12.659749+0000 mgr.x (mgr.34616) 6 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTING 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:12.775505+0000 mgr.x (mgr.34616) 7 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:12.777979+0000 mgr.x (mgr.34616) 8 : cephadm [ERR] [17/Sep/2024:08:56:12] ENGINE Error in HTTPServer.serve 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Traceback (most recent call last): 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T08:56:13.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._connections.run(self.expiration_interval) 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._run(expiration_interval) 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s = self.context.wrap_socket( 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: return self.sslsocket_class._create( 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self.do_handshake() 2024-09-17T08:56:13.230 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._sslobj.do_handshake() 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:12.876487+0000 mgr.x (mgr.34616) 9 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:12.876724+0000 mgr.x (mgr.34616) 10 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTED 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:12.077992+0000 mon.a (mon.0) 643 : cluster [DBG] mgrmap e18: x(active, since 2s), standbys: y 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:12.087249+0000 mgr.x (mgr.34616) 5 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:12.659749+0000 mgr.x (mgr.34616) 6 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTING 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:12.775505+0000 mgr.x (mgr.34616) 7 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:12.777979+0000 mgr.x (mgr.34616) 8 : cephadm [ERR] [17/Sep/2024:08:56:12] ENGINE Error in HTTPServer.serve 2024-09-17T08:56:13.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: Traceback (most recent call last): 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._connections.run(self.expiration_interval) 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._run(expiration_interval) 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T08:56:13.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s = self.context.wrap_socket( 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: return self.sslsocket_class._create( 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self.do_handshake() 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._sslobj.do_handshake() 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: 2024-09-17T08:56:13.233 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:12.876487+0000 mgr.x (mgr.34616) 9 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T08:56:13.234 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:12.876724+0000 mgr.x (mgr.34616) 10 : cephadm [INF] [17/Sep/2024:08:56:12] ENGINE Bus STARTED 2024-09-17T08:56:14.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:13.030302+0000 mgr.x (mgr.34616) 11 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:14.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:13.233427+0000 mon.b (mon.2) 47 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:14.222 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:13.233840+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:14.223 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:13.030302+0000 mgr.x (mgr.34616) 11 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:14.223 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:13.233427+0000 mon.b (mon.2) 47 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:14.223 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:13.233840+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:14.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:13.030302+0000 mgr.x (mgr.34616) 11 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:14.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:13.233427+0000 mon.b (mon.2) 47 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:14.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:13.233840+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:14.092051+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:14.092153+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-17T08:56:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:14.093031+0000 mon.b (mon.2) 48 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:14.093614+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:14.092051+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:14.092153+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:14.093031+0000 mon.b (mon.2) 48 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:14.093614+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:14.092051+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:14.092153+0000 mon.a (mon.0) 646 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:14.093031+0000 mon.b (mon.2) 48 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:14.093614+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T08:56:16.372 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:15.030678+0000 mgr.x (mgr.34616) 12 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:16.373 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:15.097642+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:16.373 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:15.097724+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-17T08:56:16.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:15.030678+0000 mgr.x (mgr.34616) 12 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:16.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:15.097642+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:16.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:15.097724+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-17T08:56:16.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:15.030678+0000 mgr.x (mgr.34616) 12 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:16.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:15.097642+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T08:56:16.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:15.097724+0000 mon.a (mon.0) 649 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-17T08:56:17.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:16.104424+0000 mon.a (mon.0) 650 : cluster [DBG] mgrmap e19: x(active, since 6s), standbys: y 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:16.143936+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:16.544849+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:16.544980+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:16.545569+0000 mon.c (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:16.547488+0000 mon.c (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:16.548555+0000 mon.c (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:16.548855+0000 mon.c (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:56:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:16.104424+0000 mon.a (mon.0) 650 : cluster [DBG] mgrmap e19: x(active, since 6s), standbys: y 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:16.143936+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:16.544849+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:16.544980+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:16.545569+0000 mon.c (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:16.547488+0000 mon.c (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:16.548555+0000 mon.c (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:56:17.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:16.548855+0000 mon.c (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:16.104424+0000 mon.a (mon.0) 650 : cluster [DBG] mgrmap e19: x(active, since 6s), standbys: y 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:16.143936+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:16.544849+0000 mon.a (mon.0) 652 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:16.544980+0000 mon.a (mon.0) 653 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:16.545569+0000 mon.c (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:16.547488+0000 mon.c (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:56:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:16.548555+0000 mon.c (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:56:17.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:16.548855+0000 mon.c (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.27:0/1771983514' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:17.031032+0000 mgr.x (mgr.34616) 13 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:17.160501+0000 mon.a (mon.0) 654 : cluster [DBG] mgrmap e20: x(active, since 7s), standbys: y 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:17.570812+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:17.575672+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:18.096143+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:18.101677+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:17.031032+0000 mgr.x (mgr.34616) 13 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:18.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:17.160501+0000 mon.a (mon.0) 654 : cluster [DBG] mgrmap e20: x(active, since 7s), standbys: y 2024-09-17T08:56:18.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:17.570812+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:17.575672+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:18.096143+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:18.101677+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:17.031032+0000 mgr.x (mgr.34616) 13 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 27 KiB data, 66 MiB used, 715 GiB / 715 GiB avail 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:17.160501+0000 mon.a (mon.0) 654 : cluster [DBG] mgrmap e20: x(active, since 7s), standbys: y 2024-09-17T08:56:18.488 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:17.570812+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:17.575672+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:18.096143+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:18.489 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:18.101677+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.870 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:18.669209+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.870 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:18.674361+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.870 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:19.170201+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:19.174891+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:19.422404+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:18.669209+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:18.674361+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:19.170201+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:19.174891+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:19.422404+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:18.669209+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:18.674361+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:19.170201+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:19.174891+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:19.959 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:19.422404+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:20.893 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:19.031636+0000 mgr.x (mgr.34616) 14 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 235 KiB/s wr, 32 op/s 2024-09-17T08:56:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:19.031636+0000 mgr.x (mgr.34616) 14 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 235 KiB/s wr, 32 op/s 2024-09-17T08:56:21.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:19.031636+0000 mgr.x (mgr.34616) 14 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 235 KiB/s wr, 32 op/s 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:20.956681+0000 mon.a (mon.0) 664 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:20.956883+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:20.957077+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:20.957229+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:21.854352+0000 mon.a (mon.0) 665 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:20.956681+0000 mon.a (mon.0) 664 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:20.956883+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:23.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:20.957077+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:20.957229+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:23.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:21.854352+0000 mon.a (mon.0) 665 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:23.009 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:20.956681+0000 mon.a (mon.0) 664 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:23.010 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:20.956883+0000 mon.b (mon.2) 49 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:23.010 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:20.957077+0000 mon.b (mon.2) 50 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:23.010 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:20.957229+0000 mon.b (mon.2) 51 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:23.010 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:21.854352+0000 mon.a (mon.0) 665 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.007 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:23 smithi027 systemd[1]: Stopping Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:24.008 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:23 smithi027 podman[49702]: 2024-09-17 08:56:23.801487001 +0000 UTC m=+0.105303056 container died e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:21.031976+0000 mgr.x (mgr.34616) 15 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 204 KiB/s wr, 27 op/s 2024-09-17T08:56:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:21.856223+0000 mon.c (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:21.856445+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:21.856765+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:21.856899+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:22.744891+0000 mon.c (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:22.748035+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:22.753834+0000 mon.b (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:22.755971+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.261233+0000 mon.b (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.264636+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.264751+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.264901+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.267404+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.273578+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.279491+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.288515+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.298019+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:21.031976+0000 mgr.x (mgr.34616) 15 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 204 KiB/s wr, 27 op/s 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:21.856223+0000 mon.c (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:21.856445+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:21.856765+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:21.856899+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:22.744891+0000 mon.c (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:22.748035+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:22.753834+0000 mon.b (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:22.755971+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.261233+0000 mon.b (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.264636+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.264751+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.264901+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.267404+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.273578+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.279491+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.288515+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.298019+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.304294+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.314065+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.323273+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.409519+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.415255+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.420867+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:23.426508+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.304294+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.314065+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.323273+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.409519+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.415255+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.420867+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:23.426508+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:21.031976+0000 mgr.x (mgr.34616) 15 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 204 KiB/s wr, 27 op/s 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:21.856223+0000 mon.c (mon.1) 28 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:21.856445+0000 mon.b (mon.2) 52 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:21.856765+0000 mon.b (mon.2) 53 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:21.856899+0000 mon.b (mon.2) 54 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:22.744891+0000 mon.c (mon.1) 29 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:22.748035+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:22.753834+0000 mon.b (mon.2) 55 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:22.755971+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.261233+0000 mon.b (mon.2) 56 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.264636+0000 mon.b (mon.2) 57 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.264751+0000 mon.b (mon.2) 58 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.264901+0000 mon.b (mon.2) 59 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.267404+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.273578+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.279491+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.288515+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.298019+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.304294+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.314065+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.323273+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.409519+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.415255+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.420867+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:23.426508+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:24.360 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49702]: 2024-09-17 08:56:24.1327902 +0000 UTC m=+0.436606246 container cleanup e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:24.360 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49702]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a 2024-09-17T08:56:24.755 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49713]: 2024-09-17 08:56:24.359879592 +0000 UTC m=+0.561687167 container remove e705ab22e1abf36d8193e7f98188f55b68bda101d03e5862cd658c9e74877423 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: Stopped Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 1.323s CPU time. 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: Starting Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49759]: 2024-09-17 08:56:24.442627769 +0000 UTC m=+0.016052844 image pull e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 docker.io/prom/node-exporter:v0.18.1 2024-09-17T08:56:24.756 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49759]: 2024-09-17 08:56:24.552465742 +0000 UTC m=+0.125890848 container create cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:25.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:23.032605+0000 mgr.x (mgr.34616) 16 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 241 KiB/s wr, 29 op/s 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:23.316477+0000 mgr.x (mgr.34616) 17 : cephadm [INF] No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:23.427425+0000 mgr.x (mgr.34616) 18 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:23.427704+0000 mgr.x (mgr.34616) 19 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi027 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:23.032605+0000 mgr.x (mgr.34616) 16 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 241 KiB/s wr, 29 op/s 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:23.316477+0000 mgr.x (mgr.34616) 17 : cephadm [INF] No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:23.427425+0000 mgr.x (mgr.34616) 18 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T08:56:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:23.427704+0000 mgr.x (mgr.34616) 19 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi027 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49759]: 2024-09-17 08:56:24.854659009 +0000 UTC m=+0.428084080 container init cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 podman[49759]: 2024-09-17 08:56:24.857569253 +0000 UTC m=+0.430994327 container start cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - arp" source="node_exporter.go:104" 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - bcache" source="node_exporter.go:104" 2024-09-17T08:56:25.009 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - bonding" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - cpu" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - edac" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - entropy" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - filefd" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2024-09-17T08:56:25.010 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - netclass" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - netdev" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - netstat" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - nfs" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - pressure" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - stat" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - textfile" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - time" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - uname" source="node_exporter.go:104" 2024-09-17T08:56:25.011 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2024-09-17T08:56:25.012 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - xfs" source="node_exporter.go:104" 2024-09-17T08:56:25.012 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg=" - zfs" source="node_exporter.go:104" 2024-09-17T08:56:25.012 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a[49774]: time="2024-09-17T08:56:24Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2024-09-17T08:56:25.012 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 bash[49759]: cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 2024-09-17T08:56:25.012 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 08:56:24 smithi027 systemd[1]: Started Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:23.032605+0000 mgr.x (mgr.34616) 16 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 241 KiB/s wr, 29 op/s 2024-09-17T08:56:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:23.316477+0000 mgr.x (mgr.34616) 17 : cephadm [INF] No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-09-17T08:56:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:23.427425+0000 mgr.x (mgr.34616) 18 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-17T08:56:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:23.427704+0000 mgr.x (mgr.34616) 19 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi027 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:24.981390+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:24.986825+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:24.988279+0000 mgr.x (mgr.34616) 20 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:24.997890+0000 mgr.x (mgr.34616) 21 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:25.032989+0000 mgr.x (mgr.34616) 22 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 215 KiB/s wr, 26 op/s 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:24.981390+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:24.986825+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:24.988279+0000 mgr.x (mgr.34616) 20 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:56:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:24.997890+0000 mgr.x (mgr.34616) 21 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:56:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:25.032989+0000 mgr.x (mgr.34616) 22 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 215 KiB/s wr, 26 op/s 2024-09-17T08:56:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:24.981390+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:24.986825+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:24.988279+0000 mgr.x (mgr.34616) 20 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-17T08:56:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:24.997890+0000 mgr.x (mgr.34616) 21 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi027 2024-09-17T08:56:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:25.032989+0000 mgr.x (mgr.34616) 22 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 215 KiB/s wr, 26 op/s 2024-09-17T08:56:29.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:27.033534+0000 mgr.x (mgr.34616) 23 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 180 KiB/s wr, 21 op/s 2024-09-17T08:56:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:27.033534+0000 mgr.x (mgr.34616) 23 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 180 KiB/s wr, 21 op/s 2024-09-17T08:56:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:27.033534+0000 mgr.x (mgr.34616) 23 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 180 KiB/s wr, 21 op/s 2024-09-17T08:56:30.853 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:29.034335+0000 mgr.x (mgr.34616) 24 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 201 KiB/s wr, 24 op/s 2024-09-17T08:56:30.853 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:29.034335+0000 mgr.x (mgr.34616) 24 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 201 KiB/s wr, 24 op/s 2024-09-17T08:56:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:29.034335+0000 mgr.x (mgr.34616) 24 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 201 KiB/s wr, 24 op/s 2024-09-17T08:56:31.952 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:31 smithi027 systemd[1]: Stopping Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:32.245 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[45065]: level=info ts=2024-09-17T08:56:31.952Z caller=main.go:536 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T08:56:32.246 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:31 smithi027 podman[50072]: 2024-09-17 08:56:31.973315831 +0000 UTC m=+0.221042348 container died 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:32.507 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50072]: 2024-09-17 08:56:32.34428763 +0000 UTC m=+0.592014135 container cleanup 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:32.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50072]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a 2024-09-17T08:56:32.869 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:31.034931+0000 mgr.x (mgr.34616) 25 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:32.869 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:31.034931+0000 mgr.x (mgr.34616) 25 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:32.869 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50084]: 2024-09-17 08:56:32.604033496 +0000 UTC m=+0.629532588 container remove 8b4e9d16b1f59cca67360f135ee9b765a7329c7f370f82aaa8043fbb6a5f387a (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:32.869 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50084]: 2024-09-17 08:56:32.679649656 +0000 UTC m=+0.705148745 volume remove 8a28b5adde083a1954b8c0a5a413d0938036d8e6535c578158e4fb8a05e243e7 2024-09-17T08:56:32.869 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T08:56:32.870 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 systemd[1]: Stopped Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:32.870 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 systemd[1]: Starting Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:33.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:31.034931+0000 mgr.x (mgr.34616) 25 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:33.162 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50126]: 2024-09-17 08:56:32.772012261 +0000 UTC m=+0.017451614 image pull 0881eb8f169f5556a292b4e2c01d683172b12830a62a9225a98a8e206bb734f0 docker.io/prom/alertmanager:v0.20.0 2024-09-17T08:56:33.163 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50126]: 2024-09-17 08:56:32.871915452 +0000 UTC m=+0.117354794 volume create b47ee2087e1864d9189c6a3493c09ce200a1f921f78c6baf262ddec48a386354 2024-09-17T08:56:33.163 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:32 smithi027 podman[50126]: 2024-09-17 08:56:32.980953384 +0000 UTC m=+0.226392728 container create b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 podman[50126]: 2024-09-17 08:56:33.258122881 +0000 UTC m=+0.503562225 container init b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 podman[50126]: 2024-09-17 08:56:33.261179478 +0000 UTC m=+0.506618820 container start b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.283Z caller=main.go:231 msg="Starting Alertmanager" version="(version=0.20.0, branch=HEAD, revision=f74be0400a6243d10bb53812d6fa408ad71ff32d)" 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.283Z caller=main.go:232 build_context="(go=go1.13.5, user=root@00c3106655f8, date=20191211-14:13:14)" 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.283Z caller=cluster.go:161 component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-09-17T08:56:33.508 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.285Z caller=cluster.go:623 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T08:56:33.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.305Z caller=coordinator.go:119 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:56:33.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.305Z caller=coordinator.go:131 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T08:56:33.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:33.307Z caller=main.go:497 msg=Listening address=:9093 2024-09-17T08:56:33.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 bash[50126]: b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc 2024-09-17T08:56:33.509 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:33 smithi027 systemd[1]: Started Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:33.035580+0000 mgr.x (mgr.34616) 26 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:33.404555+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:33.409397+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:33.410921+0000 mgr.x (mgr.34616) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:56:34.729 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:33.634574+0000 mgr.x (mgr.34616) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:56:34.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:33.035580+0000 mgr.x (mgr.34616) 26 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:34.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:33.404555+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:33.409397+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:33.410921+0000 mgr.x (mgr.34616) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:56:34.730 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:33.634574+0000 mgr.x (mgr.34616) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:56:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:33.035580+0000 mgr.x (mgr.34616) 26 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 65 KiB/s wr, 5 op/s 2024-09-17T08:56:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:33.404555+0000 mon.a (mon.0) 682 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:33.409397+0000 mon.a (mon.0) 683 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:33.410921+0000 mgr.x (mgr.34616) 27 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T08:56:34.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:33.634574+0000 mgr.x (mgr.34616) 28 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T08:56:35.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:35.285Z caller=cluster.go:648 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000119404s 2024-09-17T08:56:36.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:35.036059+0000 mgr.x (mgr.34616) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:36.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:35.036059+0000 mgr.x (mgr.34616) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:36.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:35.036059+0000 mgr.x (mgr.34616) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:39.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:37.036468+0000 mgr.x (mgr.34616) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:37.036468+0000 mgr.x (mgr.34616) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:39.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:37.036468+0000 mgr.x (mgr.34616) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 22 KiB/s wr, 2 op/s 2024-09-17T08:56:41.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:39.037414+0000 mgr.x (mgr.34616) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-17T08:56:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:39.037414+0000 mgr.x (mgr.34616) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-17T08:56:41.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:39.037414+0000 mgr.x (mgr.34616) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 22 KiB/s wr, 3 op/s 2024-09-17T08:56:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:41.037942+0000 mgr.x (mgr.34616) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:41.037942+0000 mgr.x (mgr.34616) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:41.037942+0000 mgr.x (mgr.34616) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:43.757 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 08:56:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T08:56:43.285Z caller=cluster.go:640 component=cluster msg="gossip settled; proceeding" elapsed=10.000604024s 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 systemd[1]: Stopping Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=warn ts=2024-09-17T08:56:43.917Z caller=main.go:524 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:56:43.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.917Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.918Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.918Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[39019]: level=info ts=2024-09-17T08:56:43.918Z caller=main.go:761 msg="See you next time!" 2024-09-17T08:56:43.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:43 smithi105 podman[43468]: 2024-09-17 08:56:43.927791099 +0000 UTC m=+0.122650013 container died d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:44.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:44 smithi105 podman[43468]: 2024-09-17 08:56:44.259055499 +0000 UTC m=+0.453914408 container cleanup d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:44.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:44 smithi105 podman[43468]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a 2024-09-17T08:56:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:43.038866+0000 mgr.x (mgr.34616) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:43.038866+0000 mgr.x (mgr.34616) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:43.038866+0000 mgr.x (mgr.34616) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:45 smithi105 podman[43480]: 2024-09-17 08:56:45.819100931 +0000 UTC m=+1.892494835 container remove d7c6a1537970bd0f82054972d2f1935d49fb604030d1b6682c9b00613e4c7a9f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:45 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:45 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:45 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 podman[43524]: 2024-09-17 08:56:45.908105419 +0000 UTC m=+0.017599002 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:56:46.220 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 podman[43524]: 2024-09-17 08:56:46.220106918 +0000 UTC m=+0.329600473 container create a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:46.472 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 podman[43524]: 2024-09-17 08:56:46.438640155 +0000 UTC m=+0.548133714 container init a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:46.472 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 podman[43524]: 2024-09-17 08:56:46.441799825 +0000 UTC m=+0.551293386 container start a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:45.039306+0000 mgr.x (mgr.34616) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:46.533030+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:46.538638+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:46.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.471Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:56:46.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.472Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.472Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.472Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.472Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.472Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.473Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.473Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.476Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.476Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.484Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.484Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.484Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=7.649298ms 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.485Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.485Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:56:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.485Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=info ts=2024-09-17T08:56:46.486Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43539]: level=error ts=2024-09-17T08:56:46.486Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 bash[43524]: a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c 2024-09-17T08:56:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:46 smithi105 podman[43556]: 2024-09-17 08:56:46.511170899 +0000 UTC m=+0.017167564 container died a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:47.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:45.039306+0000 mgr.x (mgr.34616) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:46.533030+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:46.538638+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:45.039306+0000 mgr.x (mgr.34616) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:46.533030+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:46.538638+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:47.117 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:47 smithi105 podman[43556]: 2024-09-17 08:56:47.117786846 +0000 UTC m=+0.623783496 container remove a119a25069049f11107f3393b6e50bb0943775eabb5dcf7036ba0d1f3b20f12c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:47.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T08:56:47.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:56:47.528 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: Stopping Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:47.528 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 podman[43667]: 2024-09-17 08:56:47.36500957 +0000 UTC m=+0.132018232 container died 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:47.997 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 podman[43667]: 2024-09-17 08:56:47.70380888 +0000 UTC m=+0.470817533 container cleanup 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:47.997 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 podman[43667]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b 2024-09-17T08:56:47.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:46.539755+0000 mgr.x (mgr.34616) 35 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T08:56:47.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:46.540030+0000 mgr.x (mgr.34616) 36 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi105 2024-09-17T08:56:48.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:46.539755+0000 mgr.x (mgr.34616) 35 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T08:56:48.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:46.540030+0000 mgr.x (mgr.34616) 36 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi105 2024-09-17T08:56:48.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:46.539755+0000 mgr.x (mgr.34616) 35 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-17T08:56:48.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:46.540030+0000 mgr.x (mgr.34616) 36 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi105 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 podman[43680]: 2024-09-17 08:56:47.996872436 +0000 UTC m=+0.635563684 container remove 9a64810c80d06da2ce4166d42664d4b81f1e6f5fba7995d6034380917dfb4b29 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: Stopped Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 1.208s CPU time. 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 systemd[1]: Starting Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 podman[43723]: 2024-09-17 08:56:48.077021271 +0000 UTC m=+0.015340518 image pull e5a616e4b9cf68dfcad7782b78e118be4310022e874d52da85c55923fb615f87 docker.io/prom/node-exporter:v0.18.1 2024-09-17T08:56:48.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 podman[43723]: 2024-09-17 08:56:48.197954219 +0000 UTC m=+0.136273453 container create 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 podman[43723]: 2024-09-17 08:56:48.399667254 +0000 UTC m=+0.337986489 container init 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 podman[43723]: 2024-09-17 08:56:48.402525221 +0000 UTC m=+0.340844464 container start 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg="Starting node_exporter (version=0.18.1, branch=HEAD, revision=3db77732e925c08f675d7404a8c46466b2ece83e)" source="node_exporter.go:156" 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg="Build context (go=go1.12.5, user=root@b50852a1acba, date=20190604-16:41:18)" source="node_exporter.go:157" 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg="Enabled collectors:" source="node_exporter.go:97" 2024-09-17T08:56:48.714 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - arp" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - bcache" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - bonding" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - conntrack" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - cpu" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - cpufreq" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - diskstats" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - edac" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - entropy" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - filefd" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - filesystem" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - hwmon" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - infiniband" source="node_exporter.go:104" 2024-09-17T08:56:48.715 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - ipvs" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - loadavg" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - mdadm" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - meminfo" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - netclass" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - netdev" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - netstat" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - nfs" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - nfsd" source="node_exporter.go:104" 2024-09-17T08:56:48.716 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - pressure" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - sockstat" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - stat" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - textfile" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - time" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - uname" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - vmstat" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - xfs" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg=" - zfs" source="node_exporter.go:104" 2024-09-17T08:56:48.717 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b[43737]: time="2024-09-17T08:56:48Z" level=info msg="Listening on :9100" source="node_exporter.go:170" 2024-09-17T08:56:48.718 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 bash[43723]: 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 2024-09-17T08:56:48.718 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 08:56:48 smithi105 systemd[1]: Started Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:47.039898+0000 mgr.x (mgr.34616) 37 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.560625+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.568446+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.617333+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.623638+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.625791+0000 mon.b (mon.2) 60 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:47.039898+0000 mgr.x (mgr.34616) 37 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.560625+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.568446+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.617333+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.623638+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.625791+0000 mon.b (mon.2) 60 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:47.039898+0000 mgr.x (mgr.34616) 37 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.560625+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.568446+0000 mon.a (mon.0) 687 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.617333+0000 mon.a (mon.0) 688 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.623638+0000 mon.a (mon.0) 689 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:56:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.625791+0000 mon.b (mon.2) 60 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:48.569636+0000 mgr.x (mgr.34616) 38 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-17T08:56:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:48.577602+0000 mgr.x (mgr.34616) 39 : cephadm [INF] Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T08:56:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:56:48.626214+0000 mgr.x (mgr.34616) 40 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:56:48.628419+0000 mgr.x (mgr.34616) 41 : cephadm [INF] Reconfiguring daemon grafana.a on smithi105 2024-09-17T08:56:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:48.569636+0000 mgr.x (mgr.34616) 38 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-17T08:56:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:48.577602+0000 mgr.x (mgr.34616) 39 : cephadm [INF] Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T08:56:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:56:48.626214+0000 mgr.x (mgr.34616) 40 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:56:48.628419+0000 mgr.x (mgr.34616) 41 : cephadm [INF] Reconfiguring daemon grafana.a on smithi105 2024-09-17T08:56:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:48.569636+0000 mgr.x (mgr.34616) 38 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-17T08:56:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:48.577602+0000 mgr.x (mgr.34616) 39 : cephadm [INF] Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T08:56:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:56:48.626214+0000 mgr.x (mgr.34616) 40 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T08:56:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:56:48.628419+0000 mgr.x (mgr.34616) 41 : cephadm [INF] Reconfiguring daemon grafana.a on smithi105 2024-09-17T08:56:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:49.040693+0000 mgr.x (mgr.34616) 42 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:49.040693+0000 mgr.x (mgr.34616) 42 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:49.040693+0000 mgr.x (mgr.34616) 42 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:51.041038+0000 mgr.x (mgr.34616) 43 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:51.041038+0000 mgr.x (mgr.34616) 43 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:53.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:51.041038+0000 mgr.x (mgr.34616) 43 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:53.041672+0000 mgr.x (mgr.34616) 44 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:53.041672+0000 mgr.x (mgr.34616) 44 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:55.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:53.041672+0000 mgr.x (mgr.34616) 44 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:56:57.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:55.042041+0000 mgr.x (mgr.34616) 45 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:57.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:55.042041+0000 mgr.x (mgr.34616) 45 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:57.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:55.042041+0000 mgr.x (mgr.34616) 45 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:57.555 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:57 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 1. 2024-09-17T08:56:57.555 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:57 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:57.555 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:57 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:58.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:56:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:57.042468+0000 mgr.x (mgr.34616) 46 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:58.857 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:58 smithi105 podman[43938]: 2024-09-17 08:56:58.442998581 +0000 UTC m=+0.449200926 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:56:59.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:56:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:57.042468+0000 mgr.x (mgr.34616) 46 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:59.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:56:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:57.042468+0000 mgr.x (mgr.34616) 46 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:56:59.115 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:56:58 smithi105 systemd[1]: Stopping Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:56:59.115 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 podman[43938]: 2024-09-17 08:56:59.015012251 +0000 UTC m=+1.021214626 container create 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:59.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 podman[43938]: 2024-09-17 08:56:59.199902534 +0000 UTC m=+1.206104873 container init 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 podman[43938]: 2024-09-17 08:56:59.202904002 +0000 UTC m=+1.209106332 container start 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.231Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.231Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.231Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.231Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.231Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.232Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:56:59.707 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.233Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.233Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.235Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.235Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.243Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.243Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.244Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.244Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=8.91715ms 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.245Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.245Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.245Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:56:59.708 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=info ts=2024-09-17T08:56:59.246Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[43991]: level=error ts=2024-09-17T08:56:59.246Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 bash[43938]: 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:56:59.709 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 podman[44008]: 2024-09-17 08:56:59.323514726 +0000 UTC m=+0.069966861 container died 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 podman[44008]: 2024-09-17 08:56:59.93779707 +0000 UTC m=+0.684276878 container remove 44005fdcaa103a25772ef9ef41b9bc5e8aa0a64876e1321cff0231d31aa40e6b (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T08:57:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:56:59 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:57:00.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-17T08:57:00.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped CleanUpService" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped provisioningServiceImpl" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped PluginManager" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped BackendPluginManager" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped UserAuthTokenService" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped Stream Manager" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped UsageStatsService" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped InternalMetricsService" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped NotificationService" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped AlertEngine" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[36390]: t=2024-09-17T08:57:00+0000 lvl=info msg="Stopped RemoteCache" logger=server reason="context canceled" 2024-09-17T08:57:00.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[43980]: 2024-09-17 08:57:00.075913434 +0000 UTC m=+1.145898213 container died 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, maintainer=Paul Cuzner , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, description=Ceph Grafana Container, io.buildah.version=1.19.8, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[43980]: 2024-09-17 08:57:00.373950566 +0000 UTC m=+1.443935345 container cleanup 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, org.label-schema.schema-version=1.0, maintainer=Paul Cuzner , io.buildah.version=1.19.8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20201204, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[43980]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[44026]: 2024-09-17 08:57:00.550006052 +0000 UTC m=+0.477289070 container remove 29288f57deb266e02d5e971029221ac94220711c0bd40979b6781c225b455e68 (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.19.8, maintainer=Paul Cuzner , org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0) 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 systemd[1]: Stopped Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:00.736 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 systemd[1]: Starting Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:57:00.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:56:59.043374+0000 mgr.x (mgr.34616) 47 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:00.997 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[44072]: 2024-09-17 08:57:00.637888995 +0000 UTC m=+0.014837084 image pull 557c83e11646f123a27b5e4b62ac6c45e7bb8b2e90d6044034d0db5b7019415c docker.io/ceph/ceph-grafana:6.7.4 2024-09-17T08:57:00.997 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[44072]: 2024-09-17 08:57:00.769447403 +0000 UTC m=+0.146395533 container create 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, org.label-schema.license=GPLv2, io.buildah.version=1.19.8, maintainer=Paul Cuzner , org.label-schema.name=CentOS Base Image, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20201204, description=Ceph Grafana Container) 2024-09-17T08:57:00.997 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[44072]: 2024-09-17 08:57:00.994165725 +0000 UTC m=+0.371113815 container init 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.license=GPLv2, maintainer=Paul Cuzner , description=Ceph Grafana Container, org.label-schema.vendor=CentOS, io.buildah.version=1.19.8) 2024-09-17T08:57:01.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:56:59.043374+0000 mgr.x (mgr.34616) 47 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:01.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:56:59.043374+0000 mgr.x (mgr.34616) 47 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:01.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:00 smithi105 podman[44072]: 2024-09-17 08:57:00.996898905 +0000 UTC m=+0.373846996 container start 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.vendor=CentOS, maintainer=Paul Cuzner , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, io.buildah.version=1.19.8, description=Ceph Grafana Container) 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Starting Grafana" logger=server version=6.7.4 commit=8e44bbc5f5 branch=HEAD compiled=2020-05-26T17:35:38+0000 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-09-17T08:57:01.279 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="App mode production" logger=settings 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing SqlStore" logger=server 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Starting DB migration" logger=migrator 2024-09-17T08:57:01.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-09-17T08:57:01.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-09-17T08:57:01.282 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-09-17T08:57:01.283 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-09-17T08:57:01.284 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-09-17T08:57:01.285 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-09-17T08:57:01.286 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-17T08:57:01.288 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-09-17T08:57:01.289 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-09-17T08:57:01.290 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-09-17T08:57:01.291 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-09-17T08:57:01.292 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-09-17T08:57:01.293 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-09-17T08:57:01.294 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-09-17T08:57:01.295 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-09-17T08:57:01.296 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-09-17T08:57:01.297 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-09-17T08:57:01.298 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-09-17T08:57:01.299 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-09-17T08:57:01.300 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-09-17T08:57:01.301 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 bash[44072]: 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-09-17T08:57:01.302 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 systemd[1]: Started Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing HTTPServer" logger=server 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing BackendPluginManager" logger=server 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing PluginManager" logger=server 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Starting plugin search" logger=plugins 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Registering plugin" logger=plugins name="Pie Chart" 2024-09-17T08:57:01.303 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Registering plugin" logger=plugins name="Status Panel" 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing HooksService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing OSSLicensingService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing InternalMetricsService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing RemoteCache" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing RenderingService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing AlertEngine" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing QuotaService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing ServerLockService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing UserAuthTokenService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing DatasourceCacheService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing LoginService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing SearchService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing TracingService" logger=server 2024-09-17T08:57:01.304 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing UsageStatsService" logger=server 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing CleanUpService" logger=server 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing NotificationService" logger=server 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing provisioningServiceImpl" logger=server 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Backend rendering via phantomJS" logger=rendering renderer=phantomJS 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=warn msg="phantomJS is deprecated and will be removed in a future release. You should consider migrating from phantomJS to grafana-image-renderer plugin. Read more at https://grafana.com/docs/grafana/latest/administration/image_rendering/" logger=rendering renderer=phantomJS 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="Initializing Stream Manager" 2024-09-17T08:57:01.305 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 08:57:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T08:57:01+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:01.043716+0000 mgr.x (mgr.34616) 48 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:01.124444+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:01.130309+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:57:01.131564+0000 mgr.x (mgr.34616) 49 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:01.131809+0000 mon.b (mon.2) 61 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:01.132159+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:01.135465+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:02.419 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:57:01.136115+0000 mgr.x (mgr.34616) 50 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:01.043716+0000 mgr.x (mgr.34616) 48 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:01.124444+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:01.130309+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:57:01.131564+0000 mgr.x (mgr.34616) 49 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:01.131809+0000 mon.b (mon.2) 61 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:01.132159+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:01.135465+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:02.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:57:01.136115+0000 mgr.x (mgr.34616) 50 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:01.043716+0000 mgr.x (mgr.34616) 48 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:01.124444+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:01.130309+0000 mon.a (mon.0) 691 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:57:01.131564+0000 mgr.x (mgr.34616) 49 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:01.131809+0000 mon.b (mon.2) 61 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:01.132159+0000 mon.a (mon.0) 692 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:01.135465+0000 mon.b (mon.2) 62 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:02.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:57:01.136115+0000 mgr.x (mgr.34616) 50 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T08:57:04.885 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:03.044604+0000 mgr.x (mgr.34616) 51 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:03.044604+0000 mgr.x (mgr.34616) 51 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:03.044604+0000 mgr.x (mgr.34616) 51 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.911847+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.917802+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.921599+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.922006+0000 mgr.x (mgr.34616) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.922750+0000 mon.b (mon.2) 64 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.923018+0000 mgr.x (mgr.34616) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.928088+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.935777+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.937040+0000 mon.b (mon.2) 66 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.942233+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.951104+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.952116+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.957545+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.965331+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.970053+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:05.008348+0000 mon.b (mon.2) 71 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:57:06.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:05.419692+0000 mon.b (mon.2) 72 : audit [DBG] from='client.? 172.21.15.105:0/1101865943' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.911847+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.917802+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.921599+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.922006+0000 mgr.x (mgr.34616) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.922750+0000 mon.b (mon.2) 64 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.923018+0000 mgr.x (mgr.34616) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.928088+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.935777+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.937040+0000 mon.b (mon.2) 66 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.942233+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.951104+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:06.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.952116+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:06.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.957545+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.965331+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:06.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.970053+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:06.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:05.008348+0000 mon.b (mon.2) 71 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:57:06.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:05.419692+0000 mon.b (mon.2) 72 : audit [DBG] from='client.? 172.21.15.105:0/1101865943' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.911847+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.917802+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.921599+0000 mon.b (mon.2) 63 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.922006+0000 mgr.x (mgr.34616) 52 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.922750+0000 mon.b (mon.2) 64 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.923018+0000 mgr.x (mgr.34616) 53 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi027.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T08:57:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.928088+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.935777+0000 mon.b (mon.2) 65 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.937040+0000 mon.b (mon.2) 66 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.942233+0000 mon.a (mon.0) 696 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.951104+0000 mon.b (mon.2) 67 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.952116+0000 mon.b (mon.2) 68 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.957545+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.965331+0000 mon.b (mon.2) 69 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.970053+0000 mon.b (mon.2) 70 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:05.008348+0000 mon.b (mon.2) 71 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:57:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:05.419692+0000 mon.b (mon.2) 72 : audit [DBG] from='client.? 172.21.15.105:0/1101865943' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T08:57:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.936160+0000 mgr.x (mgr.34616) 54 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.937322+0000 mgr.x (mgr.34616) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.951480+0000 mgr.x (mgr.34616) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.952365+0000 mgr.x (mgr.34616) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.965782+0000 mgr.x (mgr.34616) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:04.970357+0000 mgr.x (mgr.34616) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:05.044918+0000 mgr.x (mgr.34616) 60 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:07.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.936160+0000 mgr.x (mgr.34616) 54 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:07.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.937322+0000 mgr.x (mgr.34616) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:07.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.951480+0000 mgr.x (mgr.34616) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:07.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.952365+0000 mgr.x (mgr.34616) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:07.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.965782+0000 mgr.x (mgr.34616) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:07.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:04.970357+0000 mgr.x (mgr.34616) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:07.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:05.044918+0000 mgr.x (mgr.34616) 60 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.936160+0000 mgr.x (mgr.34616) 54 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.937322+0000 mgr.x (mgr.34616) 55 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.951480+0000 mgr.x (mgr.34616) 56 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.952365+0000 mgr.x (mgr.34616) 57 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.965782+0000 mgr.x (mgr.34616) 58 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:04.970357+0000 mgr.x (mgr.34616) 59 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T08:57:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:05.044918+0000 mgr.x (mgr.34616) 60 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:07.045276+0000 mgr.x (mgr.34616) 61 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:08.910767+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:07.045276+0000 mgr.x (mgr.34616) 61 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:08.910767+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:07.045276+0000 mgr.x (mgr.34616) 61 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:08.910767+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:10.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:08.916644+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:08.916644+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:08.916644+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:10.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:10 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 2. 2024-09-17T08:57:10.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:10 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:10.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:10 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:57:11.162 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45174]: 2024-09-17 08:57:10.9074387 +0000 UTC m=+0.347412187 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:57:11.162 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45174]: 2024-09-17 08:57:11.03114506 +0000 UTC m=+0.471118548 container create e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:11.162 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:09.046017+0000 mgr.x (mgr.34616) 62 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:11.162 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:10.833233+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.162 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:10.838730+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.162 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:10.839941+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:11.162 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:10.840530+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:57:11.163 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:57:10.846865+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:09.046017+0000 mgr.x (mgr.34616) 62 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:10.833233+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:10.838730+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:10.839941+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:11.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:10.840530+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:57:11.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:57:10.846865+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:09.046017+0000 mgr.x (mgr.34616) 62 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:10.833233+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:10.838730+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:10.839941+0000 mon.b (mon.2) 73 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:10.840530+0000 mon.b (mon.2) 74 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:57:11.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:57:10.846865+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:57:11.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45174]: 2024-09-17 08:57:11.274620234 +0000 UTC m=+0.714593722 container init e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:11.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45174]: 2024-09-17 08:57:11.277480257 +0000 UTC m=+0.717453747 container start e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:11.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:57:11.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:57:11.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.307Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.308Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.308Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.311Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.311Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.319Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.320Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.320Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.320Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.320Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=9.508602ms 2024-09-17T08:57:11.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=info ts=2024-09-17T08:57:11.322Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:57:11.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45232]: level=error ts=2024-09-17T08:57:11.323Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T08:57:11.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 bash[45174]: e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 2024-09-17T08:57:11.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:11.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45249]: 2024-09-17 08:57:11.365556149 +0000 UTC m=+0.034919850 container died e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:57:10.841026+0000 mgr.x (mgr.34616) 63 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:10.841430+0000 mgr.x (mgr.34616) 64 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:11.836902+0000 mon.a (mon.0) 703 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:57:10.841026+0000 mgr.x (mgr.34616) 63 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:10.841430+0000 mgr.x (mgr.34616) 64 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:11.836902+0000 mon.a (mon.0) 703 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T08:57:12.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 podman[45249]: 2024-09-17 08:57:11.953651318 +0000 UTC m=+0.623015009 container remove e2c7106535ac876d4db3e011351390450529c363b66e0cc6b0a7ee124d9ab574 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:12.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T08:57:12.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:11 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:57:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:57:10.841026+0000 mgr.x (mgr.34616) 63 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:57:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:10.841430+0000 mgr.x (mgr.34616) 64 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:11.836902+0000 mon.a (mon.0) 703 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T08:57:14.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:12.842386+0000 mgr.x (mgr.34616) 65 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:12.842386+0000 mgr.x (mgr.34616) 65 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:12.842386+0000 mgr.x (mgr.34616) 65 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:16.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:14.842851+0000 mgr.x (mgr.34616) 66 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:14.842851+0000 mgr.x (mgr.34616) 66 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:14.842851+0000 mgr.x (mgr.34616) 66 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:18.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:16.843360+0000 mgr.x (mgr.34616) 67 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:16.843360+0000 mgr.x (mgr.34616) 67 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:16.843360+0000 mgr.x (mgr.34616) 67 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-09-17T08:57:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:18.844310+0000 mgr.x (mgr.34616) 68 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:18.844310+0000 mgr.x (mgr.34616) 68 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:18.844310+0000 mgr.x (mgr.34616) 68 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T08:57:22.213 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:20.844922+0000 mgr.x (mgr.34616) 69 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:22.213 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:22 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 3. 2024-09-17T08:57:22.213 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:22 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:22.214 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:22 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:57:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:20.844922+0000 mgr.x (mgr.34616) 69 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:20.844922+0000 mgr.x (mgr.34616) 69 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 781 B/s rd, 0 op/s 2024-09-17T08:57:22.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:22 smithi105 podman[45289]: 2024-09-17 08:57:22.117163177 +0000 UTC m=+0.017840759 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:57:23.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:22.845680+0000 mgr.x (mgr.34616) 70 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:23.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:22.845680+0000 mgr.x (mgr.34616) 70 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:23.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:22.845680+0000 mgr.x (mgr.34616) 70 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:23.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:23 smithi105 podman[45289]: 2024-09-17 08:57:23.473611299 +0000 UTC m=+1.374288867 container create ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:25.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 podman[45289]: 2024-09-17 08:57:25.31350991 +0000 UTC m=+3.214187477 container init ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:25.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 podman[45289]: 2024-09-17 08:57:25.317082013 +0000 UTC m=+3.217759581 container start ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:25.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.350Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.351Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.351Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.354Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.354Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.362Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.363Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-09-17T08:57:25.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.363Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.363Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.363Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.363Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=9.390526ms 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:57:25.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:57:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:57:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:57:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=info ts=2024-09-17T08:57:25.365Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:57:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45315]: level=error ts=2024-09-17T08:57:25.366Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T08:57:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:24.846232+0000 mgr.x (mgr.34616) 71 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:24.846232+0000 mgr.x (mgr.34616) 71 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:24.846232+0000 mgr.x (mgr.34616) 71 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:26.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:26 smithi105 bash[45289]: ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 2024-09-17T08:57:26.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:26 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:26.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:26 smithi105 podman[45333]: 2024-09-17 08:57:26.321766795 +0000 UTC m=+0.948205788 container died ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:28.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:26.846587+0000 mgr.x (mgr.34616) 72 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:28.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:26.846587+0000 mgr.x (mgr.34616) 72 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:26.846587+0000 mgr.x (mgr.34616) 72 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:28.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:28 smithi105 podman[45333]: 2024-09-17 08:57:28.007193522 +0000 UTC m=+2.633632513 container remove ce2b460f1f224e6e43db77290a25384ab78100c875f59bc8b673e9f20d7468c3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:28.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:28 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T08:57:28.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:28 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:57:30.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:28.847367+0000 mgr.x (mgr.34616) 73 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:28.847367+0000 mgr.x (mgr.34616) 73 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:28.847367+0000 mgr.x (mgr.34616) 73 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:32.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:30.847920+0000 mgr.x (mgr.34616) 74 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:30.847920+0000 mgr.x (mgr.34616) 74 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:30.847920+0000 mgr.x (mgr.34616) 74 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:32.848699+0000 mgr.x (mgr.34616) 75 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:32.848699+0000 mgr.x (mgr.34616) 75 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:32.848699+0000 mgr.x (mgr.34616) 75 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:36.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:34.849319+0000 mgr.x (mgr.34616) 76 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:34.849319+0000 mgr.x (mgr.34616) 76 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:34.849319+0000 mgr.x (mgr.34616) 76 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:38.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:36.849968+0000 mgr.x (mgr.34616) 77 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:38.212 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 4. 2024-09-17T08:57:38.212 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:38.213 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:57:38.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:36.849968+0000 mgr.x (mgr.34616) 77 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:36.849968+0000 mgr.x (mgr.34616) 77 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 podman[45374]: 2024-09-17 08:57:38.116051246 +0000 UTC m=+0.018190436 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 podman[45374]: 2024-09-17 08:57:38.224327152 +0000 UTC m=+0.126466325 container create 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 podman[45374]: 2024-09-17 08:57:38.426039313 +0000 UTC m=+0.328178492 container init 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 podman[45374]: 2024-09-17 08:57:38.42901278 +0000 UTC m=+0.331151963 container start 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.458Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.458Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.459Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.459Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T08:57:38.463 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.459Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T08:57:38.464 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.459Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T08:57:38.464 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.460Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T08:57:38.464 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.460Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T08:57:38.464 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.462Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T08:57:38.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.464Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-09-17T08:57:38.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.472Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-09-17T08:57:38.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.472Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-09-17T08:57:38.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.472Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.473Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.473Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.473Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.473Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=10.765011ms 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.475Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.475Z caller=main.go:695 msg="TSDB started" 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T08:57:38.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=info ts=2024-09-17T08:57:38.476Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[45388]: level=error ts=2024-09-17T08:57:38.476Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 bash[45374]: 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:38.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:38 smithi105 podman[45406]: 2024-09-17 08:57:38.55903347 +0000 UTC m=+0.070309555 container died 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:39.227 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:39 smithi105 podman[45406]: 2024-09-17 08:57:39.021806913 +0000 UTC m=+0.533082991 container remove 77f9bf67d18092532f0932ee6d0216859fc66ef5dd1c5162094edcf98b61d178 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T08:57:39.227 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:39 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T08:57:39.227 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:39 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:57:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:38.850654+0000 mgr.x (mgr.34616) 78 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:38.850654+0000 mgr.x (mgr.34616) 78 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:38.850654+0000 mgr.x (mgr.34616) 78 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:40.851141+0000 mgr.x (mgr.34616) 79 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:40.851141+0000 mgr.x (mgr.34616) 79 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:40.851141+0000 mgr.x (mgr.34616) 79 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:44.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:42.851911+0000 mgr.x (mgr.34616) 80 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:44.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:42.851911+0000 mgr.x (mgr.34616) 80 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:42.851911+0000 mgr.x (mgr.34616) 80 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:44.852326+0000 mgr.x (mgr.34616) 81 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:44.852326+0000 mgr.x (mgr.34616) 81 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:46.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:44.852326+0000 mgr.x (mgr.34616) 81 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:48.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:46.852880+0000 mgr.x (mgr.34616) 82 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:46.852880+0000 mgr.x (mgr.34616) 82 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:46.852880+0000 mgr.x (mgr.34616) 82 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:49.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:49 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 5. 2024-09-17T08:57:49.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:49 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:49.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:49 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Start request repeated too quickly. 2024-09-17T08:57:49.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:49 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T08:57:49.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 08:57:49 smithi105 systemd[1]: Failed to start Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:57:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:48.853700+0000 mgr.x (mgr.34616) 83 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:50.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:48.853700+0000 mgr.x (mgr.34616) 83 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:48.853700+0000 mgr.x (mgr.34616) 83 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:52.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:50.854094+0000 mgr.x (mgr.34616) 84 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:50.854094+0000 mgr.x (mgr.34616) 84 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:50.854094+0000 mgr.x (mgr.34616) 84 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:54.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:52.855085+0000 mgr.x (mgr.34616) 85 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:52.855085+0000 mgr.x (mgr.34616) 85 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:52.855085+0000 mgr.x (mgr.34616) 85 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:57:56.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:54.855533+0000 mgr.x (mgr.34616) 86 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:54.855533+0000 mgr.x (mgr.34616) 86 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:54.855533+0000 mgr.x (mgr.34616) 86 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:56.856031+0000 mgr.x (mgr.34616) 87 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:56.856031+0000 mgr.x (mgr.34616) 87 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:57:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:56.856031+0000 mgr.x (mgr.34616) 87 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:57:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:57:58.856803+0000 mgr.x (mgr.34616) 88 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:57:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:57:58.856803+0000 mgr.x (mgr.34616) 88 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:57:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:57:58.856803+0000 mgr.x (mgr.34616) 88 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:00.857287+0000 mgr.x (mgr.34616) 89 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:00.857287+0000 mgr.x (mgr.34616) 89 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:00.857287+0000 mgr.x (mgr.34616) 89 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:04.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:02.858181+0000 mgr.x (mgr.34616) 90 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:02.858181+0000 mgr.x (mgr.34616) 90 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:04.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:02.858181+0000 mgr.x (mgr.34616) 90 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:04.858618+0000 mgr.x (mgr.34616) 91 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:06.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:04.858618+0000 mgr.x (mgr.34616) 91 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:06.367 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:04.858618+0000 mgr.x (mgr.34616) 91 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:06.859224+0000 mgr.x (mgr.34616) 92 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:06.859224+0000 mgr.x (mgr.34616) 92 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:08.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:06.859224+0000 mgr.x (mgr.34616) 92 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:08.860090+0000 mgr.x (mgr.34616) 93 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:08.860090+0000 mgr.x (mgr.34616) 93 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:08.860090+0000 mgr.x (mgr.34616) 93 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:12.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:10.860658+0000 mgr.x (mgr.34616) 94 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:10.958471+0000 mon.b (mon.2) 75 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:58:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:10.860658+0000 mgr.x (mgr.34616) 94 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:10.958471+0000 mon.b (mon.2) 75 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:58:12.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:10.860658+0000 mgr.x (mgr.34616) 94 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:10.958471+0000 mon.b (mon.2) 75 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:58:14.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:12.861318+0000 mgr.x (mgr.34616) 95 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:12.861318+0000 mgr.x (mgr.34616) 95 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:12.861318+0000 mgr.x (mgr.34616) 95 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:14.847965+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:14.853001+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:14.861786+0000 mgr.x (mgr.34616) 96 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:14.847965+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:14.853001+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:14.861786+0000 mgr.x (mgr.34616) 96 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:14.847965+0000 mon.a (mon.0) 704 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:14.853001+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:14.861786+0000 mgr.x (mgr.34616) 96 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:16.862365+0000 mgr.x (mgr.34616) 97 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:16.925809+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:16.931630+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:17.225839+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:17.226452+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:58:17.232715+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:16.862365+0000 mgr.x (mgr.34616) 97 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:16.925809+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:16.931630+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:17.225839+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:58:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:17.226452+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:58:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:58:17.232715+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:16.862365+0000 mgr.x (mgr.34616) 97 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:16.925809+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:16.931630+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:17.225839+0000 mon.b (mon.2) 76 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:17.226452+0000 mon.b (mon.2) 77 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:58:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:58:17.232715+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:58:17.226931+0000 mgr.x (mgr.34616) 98 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:17.227582+0000 mgr.x (mgr.34616) 99 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:17.227739+0000 mgr.x (mgr.34616) 100 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:58:17.226931+0000 mgr.x (mgr.34616) 98 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:17.227582+0000 mgr.x (mgr.34616) 99 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T08:58:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:17.227739+0000 mgr.x (mgr.34616) 100 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:58:17.226931+0000 mgr.x (mgr.34616) 98 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:58:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:17.227582+0000 mgr.x (mgr.34616) 99 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-09-17T08:58:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:17.227739+0000 mgr.x (mgr.34616) 100 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:19.228192+0000 mgr.x (mgr.34616) 101 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:19.228192+0000 mgr.x (mgr.34616) 101 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:19.228192+0000 mgr.x (mgr.34616) 101 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:21.228691+0000 mgr.x (mgr.34616) 102 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-09-17T08:58:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:21.228691+0000 mgr.x (mgr.34616) 102 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-09-17T08:58:22.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:21.228691+0000 mgr.x (mgr.34616) 102 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 611 B/s rd, 0 op/s 2024-09-17T08:58:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:23.229564+0000 mgr.x (mgr.34616) 103 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:23.229564+0000 mgr.x (mgr.34616) 103 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:23.229564+0000 mgr.x (mgr.34616) 103 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:27.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:25.230003+0000 mgr.x (mgr.34616) 104 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:25.230003+0000 mgr.x (mgr.34616) 104 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:27.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:25.230003+0000 mgr.x (mgr.34616) 104 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:29.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:27.230767+0000 mgr.x (mgr.34616) 105 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:58:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:27.230767+0000 mgr.x (mgr.34616) 105 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:58:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:27.230767+0000 mgr.x (mgr.34616) 105 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:58:31.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:29.231357+0000 mgr.x (mgr.34616) 106 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:29.231357+0000 mgr.x (mgr.34616) 106 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:29.231357+0000 mgr.x (mgr.34616) 106 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:31.231942+0000 mgr.x (mgr.34616) 107 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:31.231942+0000 mgr.x (mgr.34616) 107 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:33.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:31.231942+0000 mgr.x (mgr.34616) 107 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:33.232653+0000 mgr.x (mgr.34616) 108 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:35.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:33.232653+0000 mgr.x (mgr.34616) 108 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:35.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:33.232653+0000 mgr.x (mgr.34616) 108 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:35.233125+0000 mgr.x (mgr.34616) 109 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:35.233125+0000 mgr.x (mgr.34616) 109 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:35.233125+0000 mgr.x (mgr.34616) 109 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:39.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:37.233932+0000 mgr.x (mgr.34616) 110 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:37.233932+0000 mgr.x (mgr.34616) 110 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:39.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:37.233932+0000 mgr.x (mgr.34616) 110 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:39.234400+0000 mgr.x (mgr.34616) 111 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:41.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:39.234400+0000 mgr.x (mgr.34616) 111 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:41.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:39.234400+0000 mgr.x (mgr.34616) 111 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:41.234860+0000 mgr.x (mgr.34616) 112 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:41.234860+0000 mgr.x (mgr.34616) 112 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:43.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:41.234860+0000 mgr.x (mgr.34616) 112 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:45.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:43.235555+0000 mgr.x (mgr.34616) 113 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:43.235555+0000 mgr.x (mgr.34616) 113 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:43.235555+0000 mgr.x (mgr.34616) 113 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:45.236161+0000 mgr.x (mgr.34616) 114 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:45.236161+0000 mgr.x (mgr.34616) 114 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:45.236161+0000 mgr.x (mgr.34616) 114 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:47.237084+0000 mgr.x (mgr.34616) 115 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:47.237084+0000 mgr.x (mgr.34616) 115 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:47.237084+0000 mgr.x (mgr.34616) 115 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:49.237559+0000 mgr.x (mgr.34616) 116 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:49.237559+0000 mgr.x (mgr.34616) 116 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:49.237559+0000 mgr.x (mgr.34616) 116 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:51.238013+0000 mgr.x (mgr.34616) 117 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:51.238013+0000 mgr.x (mgr.34616) 117 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:51.238013+0000 mgr.x (mgr.34616) 117 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:53.238851+0000 mgr.x (mgr.34616) 118 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:53.238851+0000 mgr.x (mgr.34616) 118 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:55.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:53.238851+0000 mgr.x (mgr.34616) 118 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:57.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:55.239253+0000 mgr.x (mgr.34616) 119 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:57.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:55.239253+0000 mgr.x (mgr.34616) 119 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:57.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:55.239253+0000 mgr.x (mgr.34616) 119 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:58:59.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:58:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:57.240182+0000 mgr.x (mgr.34616) 120 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:58:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:57.240182+0000 mgr.x (mgr.34616) 120 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:58:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:58:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:57.240182+0000 mgr.x (mgr.34616) 120 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:58:59.240659+0000 mgr.x (mgr.34616) 121 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:58:59.240659+0000 mgr.x (mgr.34616) 121 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:58:59.240659+0000 mgr.x (mgr.34616) 121 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:01.241118+0000 mgr.x (mgr.34616) 122 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:01.241118+0000 mgr.x (mgr.34616) 122 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:01.241118+0000 mgr.x (mgr.34616) 122 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:03.241864+0000 mgr.x (mgr.34616) 123 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:03.241864+0000 mgr.x (mgr.34616) 123 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:03.241864+0000 mgr.x (mgr.34616) 123 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:05.242311+0000 mgr.x (mgr.34616) 124 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:05.242311+0000 mgr.x (mgr.34616) 124 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:05.242311+0000 mgr.x (mgr.34616) 124 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:07.243170+0000 mgr.x (mgr.34616) 125 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:07.243170+0000 mgr.x (mgr.34616) 125 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:09.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:07.243170+0000 mgr.x (mgr.34616) 125 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:09.243682+0000 mgr.x (mgr.34616) 126 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:09.243682+0000 mgr.x (mgr.34616) 126 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:09.243682+0000 mgr.x (mgr.34616) 126 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:13.059 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:11.244125+0000 mgr.x (mgr.34616) 127 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:13.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:11.244125+0000 mgr.x (mgr.34616) 127 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:11.244125+0000 mgr.x (mgr.34616) 127 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:14.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T08:59:15.086 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:13.244878+0000 mgr.x (mgr.34616) 128 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:15.087 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:13.244878+0000 mgr.x (mgr.34616) 128 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:13.244878+0000 mgr.x (mgr.34616) 128 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:17.085 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:15.245415+0000 mgr.x (mgr.34616) 129 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:17.085 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:16.710311+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.27:0/1836284581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:59:17.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:15.245415+0000 mgr.x (mgr.34616) 129 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:17.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:16.710311+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.27:0/1836284581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:59:17.086 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled to redeploy mgr.y on host 'smithi027' 2024-09-17T08:59:17.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:15.245415+0000 mgr.x (mgr.34616) 129 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:17.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:16.710311+0000 mon.c (mon.1) 30 : audit [DBG] from='client.? 172.21.15.27:0/1836284581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T08:59:17.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps --refresh' 2024-09-17T08:59:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:17.064226+0000 mgr.x (mgr.34616) 130 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:17.069070+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:59:17.070944+0000 mgr.x (mgr.34616) 131 : cephadm [INF] Schedule redeploy daemon mgr.y 2024-09-17T08:59:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:17.077013+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:17.082816+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:17.084602+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:17.246048+0000 mgr.x (mgr.34616) 132 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:17.064226+0000 mgr.x (mgr.34616) 130 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:17.069070+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:59:17.070944+0000 mgr.x (mgr.34616) 131 : cephadm [INF] Schedule redeploy daemon mgr.y 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:17.077013+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:17.082816+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:17.084602+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:18.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:17.246048+0000 mgr.x (mgr.34616) 132 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:17.064226+0000 mgr.x (mgr.34616) 130 : audit [DBG] from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:17.069070+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:59:17.070944+0000 mgr.x (mgr.34616) 131 : cephadm [INF] Schedule redeploy daemon mgr.y 2024-09-17T08:59:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:17.077013+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:17.082816+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:18.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:17.084602+0000 mon.b (mon.2) 78 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:18.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:17.246048+0000 mgr.x (mgr.34616) 132 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:19.998 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:18.927823+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:19.998 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:18.927823+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:18.927823+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:20.724 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 65s ago 10m 9.97M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (2m) 63s ago 10m 27.6M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (2m) 63s ago 9m 40.8M - 3.4 24ecd6d5f14c 9223e1c260cd 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443 running (5m) 63s ago 14m 480M - 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:9283 running (15m) 65s ago 15m 359M - 16.2.0 24ecd6d5f14c 564f13fefc6d 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (15m) 65s ago 15m 55.4M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (14m) 63s ago 14m 49.4M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (14m) 65s ago 14m 44.5M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 65s ago 11m 9328k - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T08:59:20.725 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (2m) 63s ago 10m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (14m) 65s ago 14m 47.1M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (13m) 65s ago 13m 47.5M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (13m) 65s ago 13m 43.3M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (13m) 65s ago 13m 46.7M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (12m) 63s ago 12m 49.4M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T08:59:20.726 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (12m) 63s ago 12m 44.6M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T08:59:20.727 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (12m) 63s ago 12m 45.5M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T08:59:20.727 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (11m) 63s ago 11m 49.9M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T08:59:20.727 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 error 63s ago 11m - - 2024-09-17T08:59:20.727 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (10m) 65s ago 10m 37.4M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T08:59:20.727 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (10m) 63s ago 10m 37.8M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T08:59:21.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:19.246420+0000 mgr.x (mgr.34616) 133 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:21.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:19.246420+0000 mgr.x (mgr.34616) 133 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:19.246420+0000 mgr.x (mgr.34616) 133 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:21.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T08:59:22.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:20.716306+0000 mgr.x (mgr.34616) 134 : audit [DBG] from='client.24814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:22.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:20.716306+0000 mgr.x (mgr.34616) 134 : audit [DBG] from='client.24814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:20.716306+0000 mgr.x (mgr.34616) 134 : audit [DBG] from='client.24814 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T08:59:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:21.246776+0000 mgr.x (mgr.34616) 135 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:21.246776+0000 mgr.x (mgr.34616) 135 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:21.246776+0000 mgr.x (mgr.34616) 135 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 82 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 0 B/s wr, 20 op/s 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:23.247425+0000 mgr.x (mgr.34616) 136 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:24.904372+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:24.910193+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:23.247425+0000 mgr.x (mgr.34616) 136 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:24.904372+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:24.910193+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:25.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:23.247425+0000 mgr.x (mgr.34616) 136 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:25.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:24.904372+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:25.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:24.910193+0000 mon.a (mon.0) 714 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.202160+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.202782+0000 mon.b (mon.2) 80 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.209344+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.286879+0000 mon.b (mon.2) 81 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.287584+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.288397+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:25.288980+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.202160+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.202782+0000 mon.b (mon.2) 80 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.209344+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.286879+0000 mon.b (mon.2) 81 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.287584+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.288397+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:59:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:25.288980+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.202160+0000 mon.b (mon.2) 79 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.202782+0000 mon.b (mon.2) 80 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.209344+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.286879+0000 mon.b (mon.2) 81 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.287584+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.34616 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.288397+0000 mon.b (mon.2) 82 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T08:59:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:25.288980+0000 mon.b (mon.2) 83 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:27.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:26 smithi027 systemd[1]: Stopping Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:59:27.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:27 smithi027 podman[52336]: 2024-09-17 08:59:27.090677059 +0000 UTC m=+0.119492251 container died 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, CEPH_POINT_RELEASE=-16.2.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Base Image, RELEASE=HEAD, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T08:59:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:59:25.203295+0000 mgr.x (mgr.34616) 137 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:25.203700+0000 mgr.x (mgr.34616) 138 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:25.203881+0000 mgr.x (mgr.34616) 139 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:59:25.289558+0000 mgr.x (mgr.34616) 140 : cephadm [INF] Deploying daemon mgr.y on smithi027 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:59:25.203295+0000 mgr.x (mgr.34616) 137 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:25.203700+0000 mgr.x (mgr.34616) 138 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:25.203881+0000 mgr.x (mgr.34616) 139 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-17T08:59:27.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:59:25.289558+0000 mgr.x (mgr.34616) 140 : cephadm [INF] Deploying daemon mgr.y on smithi027 2024-09-17T08:59:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:59:25.203295+0000 mgr.x (mgr.34616) 137 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:25.203700+0000 mgr.x (mgr.34616) 138 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:25.203881+0000 mgr.x (mgr.34616) 139 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 0 B/s wr, 144 op/s 2024-09-17T08:59:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:59:25.289558+0000 mgr.x (mgr.34616) 140 : cephadm [INF] Deploying daemon mgr.y on smithi027 2024-09-17T08:59:27.721 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:27 smithi027 podman[52336]: 2024-09-17 08:59:27.469540468 +0000 UTC m=+0.498355647 container cleanup 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, RELEASE=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=-16.2.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20201204, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0) 2024-09-17T08:59:27.721 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:27 smithi027 bash[52336]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y 2024-09-17T08:59:27.996 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:27 smithi027 podman[52348]: 2024-09-17 08:59:27.720924916 +0000 UTC m=+0.624071936 container remove 564f13fefc6deb6e473da17a82aef389216d0cd9e8d383fba7901c70e84125b8 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.y, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, ceph=True, maintainer=Dimitri Savineau , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Base Image, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD) 2024-09-17T08:59:27.996 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:27 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-17T08:59:28.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-17T08:59:28.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 systemd[1]: Stopped Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:59:28.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 systemd[1]: Starting Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T08:59:28.694 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 podman[52493]: 2024-09-17 08:59:28.334292115 +0000 UTC m=+0.016652855 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T08:59:28.694 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 podman[52493]: 2024-09-17 08:59:28.455943582 +0000 UTC m=+0.138304359 container create f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T08:59:28.694 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 podman[52493]: 2024-09-17 08:59:28.666567614 +0000 UTC m=+0.348928345 container init f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T08:59:28.694 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 podman[52493]: 2024-09-17 08:59:28.669370531 +0000 UTC m=+0.351731266 container start f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T08:59:28.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-mgr[52513]: -- 172.21.15.27:0/3352165446 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555d28ba2d00 con 0x555d28b77400 2024-09-17T08:59:28.949 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 bash[52493]: f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 2024-09-17T08:59:28.950 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 systemd[1]: Started Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T08:59:28.950 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:28.791+0000 7fcd72095040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T08:59:28.950 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:28.854+0000 7fcd72095040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T08:59:29.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:27.204609+0000 mgr.x (mgr.34616) 141 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:29.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:28.847366+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:28.853555+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:28.861251+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:28.870850+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:28.908472+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.015+0000 7fcd72095040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.085+0000 7fcd72095040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T08:59:29.252 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.250+0000 7fcd72095040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:27.204609+0000 mgr.x (mgr.34616) 141 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:28.847366+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:28.853555+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:28.861251+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:28.870850+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:28.908472+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:27.204609+0000 mgr.x (mgr.34616) 141 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:28.847366+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:28.853555+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:28.861251+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:28.870850+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:28.908472+0000 mon.b (mon.2) 84 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T08:59:29.507 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.407+0000 7fcd72095040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T08:59:29.802 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.739+0000 7fcd72095040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T08:59:30.172 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:29.204980+0000 mgr.x (mgr.34616) 142 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:30.173 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:29.802+0000 7fcd72095040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T08:59:30.173 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:30.001+0000 7fcd72095040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T08:59:30.173 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:29.204980+0000 mgr.x (mgr.34616) 142 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:29.204980+0000 mgr.x (mgr.34616) 142 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:31.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:30.959+0000 7fcd72095040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T08:59:31.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:31.033+0000 7fcd72095040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T08:59:32.118 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:31.844+0000 7fcd72095040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T08:59:32.119 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:31.968+0000 7fcd72095040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T08:59:32.119 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.118+0000 7fcd72095040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T08:59:32.420 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:31.205293+0000 mgr.x (mgr.34616) 143 : cluster [DBG] pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:32.420 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:31.205293+0000 mgr.x (mgr.34616) 143 : cluster [DBG] pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:32.420 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.182+0000 7fcd72095040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T08:59:32.420 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.340+0000 7fcd72095040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T08:59:32.421 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.419+0000 7fcd72095040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T08:59:32.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:31.205293+0000 mgr.x (mgr.34616) 143 : cluster [DBG] pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-09-17T08:59:32.727 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.488+0000 7fcd72095040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T08:59:32.727 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.653+0000 7fcd72095040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T08:59:32.727 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.726+0000 7fcd72095040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T08:59:33.007 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.796+0000 7fcd72095040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T08:59:33.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.882+0000 7fcd72095040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T08:59:33.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:32.946+0000 7fcd72095040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T08:59:33.507 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:33.007+0000 7fcd72095040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T08:59:34.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 08:59:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T08:59:33.843+0000 7fcd72095040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:33.206148+0000 mgr.x (mgr.34616) 144 : cluster [DBG] pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.305891+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.312855+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:33.845500+0000 mon.a (mon.0) 723 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:33.845631+0000 mon.a (mon.0) 724 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.847000+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.847388+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.848118+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:59:34.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:33.848431+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:33.206148+0000 mgr.x (mgr.34616) 144 : cluster [DBG] pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.305891+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.312855+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:33.845500+0000 mon.a (mon.0) 723 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:33.845631+0000 mon.a (mon.0) 724 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.847000+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:59:34.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.847388+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.848118+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:33.848431+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:33.206148+0000 mgr.x (mgr.34616) 144 : cluster [DBG] pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.305891+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.312855+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:33.845500+0000 mon.a (mon.0) 723 : cluster [DBG] Standby manager daemon y restarted 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:33.845631+0000 mon.a (mon.0) 724 : cluster [DBG] Standby manager daemon y started 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.847000+0000 mon.a (mon.0) 725 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.847388+0000 mon.a (mon.0) 726 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.848118+0000 mon.a (mon.0) 727 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T08:59:34.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:33.848431+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.? 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:34.735197+0000 mon.a (mon.0) 729 : cluster [DBG] mgrmap e21: x(active, since 3m), standbys: y 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:34.878230+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:34.886286+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:34.887456+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:34.888041+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T08:59:34.888519+0000 mgr.x (mgr.34616) 145 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:34.889021+0000 mgr.x (mgr.34616) 146 : cluster [DBG] pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:34.889198+0000 mgr.x (mgr.34616) 147 : cluster [DBG] pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T08:59:34.893923+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:34.735197+0000 mon.a (mon.0) 729 : cluster [DBG] mgrmap e21: x(active, since 3m), standbys: y 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:34.878230+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:34.886286+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:34.887456+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:34.888041+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T08:59:34.888519+0000 mgr.x (mgr.34616) 145 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:34.889021+0000 mgr.x (mgr.34616) 146 : cluster [DBG] pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:34.889198+0000 mgr.x (mgr.34616) 147 : cluster [DBG] pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T08:59:34.893923+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:34.735197+0000 mon.a (mon.0) 729 : cluster [DBG] mgrmap e21: x(active, since 3m), standbys: y 2024-09-17T08:59:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:34.878230+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:34.886286+0000 mon.a (mon.0) 731 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:34.887456+0000 mon.b (mon.2) 85 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T08:59:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:34.888041+0000 mon.b (mon.2) 86 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T08:59:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T08:59:34.888519+0000 mgr.x (mgr.34616) 145 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T08:59:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:34.889021+0000 mgr.x (mgr.34616) 146 : cluster [DBG] pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:34.889198+0000 mgr.x (mgr.34616) 147 : cluster [DBG] pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T08:59:34.893923+0000 mon.a (mon.0) 732 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T08:59:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:36.889654+0000 mgr.x (mgr.34616) 148 : cluster [DBG] pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 528 B/s rd, 0 op/s 2024-09-17T08:59:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:36.889654+0000 mgr.x (mgr.34616) 148 : cluster [DBG] pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 528 B/s rd, 0 op/s 2024-09-17T08:59:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:36.889654+0000 mgr.x (mgr.34616) 148 : cluster [DBG] pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 528 B/s rd, 0 op/s 2024-09-17T08:59:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:38.890607+0000 mgr.x (mgr.34616) 149 : cluster [DBG] pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:38.890607+0000 mgr.x (mgr.34616) 149 : cluster [DBG] pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:38.890607+0000 mgr.x (mgr.34616) 149 : cluster [DBG] pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:40.891171+0000 mgr.x (mgr.34616) 150 : cluster [DBG] pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:40.891171+0000 mgr.x (mgr.34616) 150 : cluster [DBG] pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:40.891171+0000 mgr.x (mgr.34616) 150 : cluster [DBG] pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:44.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:42.892054+0000 mgr.x (mgr.34616) 151 : cluster [DBG] pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:44.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:42.892054+0000 mgr.x (mgr.34616) 151 : cluster [DBG] pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:42.892054+0000 mgr.x (mgr.34616) 151 : cluster [DBG] pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-09-17T08:59:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:44.892653+0000 mgr.x (mgr.34616) 152 : cluster [DBG] pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:59:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:44.892653+0000 mgr.x (mgr.34616) 152 : cluster [DBG] pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:59:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:44.892653+0000 mgr.x (mgr.34616) 152 : cluster [DBG] pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T08:59:48.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:46.893141+0000 mgr.x (mgr.34616) 153 : cluster [DBG] pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:46.893141+0000 mgr.x (mgr.34616) 153 : cluster [DBG] pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:48.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:46.893141+0000 mgr.x (mgr.34616) 153 : cluster [DBG] pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:48.894096+0000 mgr.x (mgr.34616) 154 : cluster [DBG] pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:48.894096+0000 mgr.x (mgr.34616) 154 : cluster [DBG] pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:48.894096+0000 mgr.x (mgr.34616) 154 : cluster [DBG] pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:51.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:50.894536+0000 mgr.x (mgr.34616) 155 : cluster [DBG] pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:50.894536+0000 mgr.x (mgr.34616) 155 : cluster [DBG] pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:50.894536+0000 mgr.x (mgr.34616) 155 : cluster [DBG] pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:53.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:52.895556+0000 mgr.x (mgr.34616) 156 : cluster [DBG] pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:54.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:52.895556+0000 mgr.x (mgr.34616) 156 : cluster [DBG] pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:52.895556+0000 mgr.x (mgr.34616) 156 : cluster [DBG] pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:54.896012+0000 mgr.x (mgr.34616) 157 : cluster [DBG] pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:54.896012+0000 mgr.x (mgr.34616) 157 : cluster [DBG] pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:54.896012+0000 mgr.x (mgr.34616) 157 : cluster [DBG] pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:58.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:56.896470+0000 mgr.x (mgr.34616) 158 : cluster [DBG] pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:56.896470+0000 mgr.x (mgr.34616) 158 : cluster [DBG] pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:58.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:56.896470+0000 mgr.x (mgr.34616) 158 : cluster [DBG] pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T08:59:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 08:59:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T08:59:58.897210+0000 mgr.x (mgr.34616) 159 : cluster [DBG] pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 08:59:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T08:59:58.897210+0000 mgr.x (mgr.34616) 159 : cluster [DBG] pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T08:59:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 08:59:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T08:59:58.897210+0000 mgr.x (mgr.34616) 159 : cluster [DBG] pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:00.000184+0000 mon.a (mon.0) 733 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:00.000230+0000 mon.a (mon.0) 734 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:00.000253+0000 mon.a (mon.0) 735 : cluster [WRN] daemon prometheus.a on smithi105 is in error state 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:00.000184+0000 mon.a (mon.0) 733 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:00.000230+0000 mon.a (mon.0) 734 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:00:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:00.000253+0000 mon.a (mon.0) 735 : cluster [WRN] daemon prometheus.a on smithi105 is in error state 2024-09-17T09:00:00.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:00.000184+0000 mon.a (mon.0) 733 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:00:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:00.000230+0000 mon.a (mon.0) 734 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:00:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:00.000253+0000 mon.a (mon.0) 735 : cluster [WRN] daemon prometheus.a on smithi105 is in error state 2024-09-17T09:00:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:00.897692+0000 mgr.x (mgr.34616) 160 : cluster [DBG] pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:00.897692+0000 mgr.x (mgr.34616) 160 : cluster [DBG] pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:01.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:00.897692+0000 mgr.x (mgr.34616) 160 : cluster [DBG] pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:04.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:02.898398+0000 mgr.x (mgr.34616) 161 : cluster [DBG] pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:02.898398+0000 mgr.x (mgr.34616) 161 : cluster [DBG] pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:04.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:02.898398+0000 mgr.x (mgr.34616) 161 : cluster [DBG] pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:04.898860+0000 mgr.x (mgr.34616) 162 : cluster [DBG] pgmap v126: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:04.898860+0000 mgr.x (mgr.34616) 162 : cluster [DBG] pgmap v126: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:04.898860+0000 mgr.x (mgr.34616) 162 : cluster [DBG] pgmap v126: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:07.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:06.899270+0000 mgr.x (mgr.34616) 163 : cluster [DBG] pgmap v127: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:06.899270+0000 mgr.x (mgr.34616) 163 : cluster [DBG] pgmap v127: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:06.899270+0000 mgr.x (mgr.34616) 163 : cluster [DBG] pgmap v127: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:08.900067+0000 mgr.x (mgr.34616) 164 : cluster [DBG] pgmap v128: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:08.900067+0000 mgr.x (mgr.34616) 164 : cluster [DBG] pgmap v128: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:08.900067+0000 mgr.x (mgr.34616) 164 : cluster [DBG] pgmap v128: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:10.900484+0000 mgr.x (mgr.34616) 165 : cluster [DBG] pgmap v129: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:10.900484+0000 mgr.x (mgr.34616) 165 : cluster [DBG] pgmap v129: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:11.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:10.900484+0000 mgr.x (mgr.34616) 165 : cluster [DBG] pgmap v129: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:12.901368+0000 mgr.x (mgr.34616) 166 : cluster [DBG] pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:12.901368+0000 mgr.x (mgr.34616) 166 : cluster [DBG] pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:12.901368+0000 mgr.x (mgr.34616) 166 : cluster [DBG] pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:14.901909+0000 mgr.x (mgr.34616) 167 : cluster [DBG] pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:14.901909+0000 mgr.x (mgr.34616) 167 : cluster [DBG] pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:14.901909+0000 mgr.x (mgr.34616) 167 : cluster [DBG] pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:16.902443+0000 mgr.x (mgr.34616) 168 : cluster [DBG] pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:16.902443+0000 mgr.x (mgr.34616) 168 : cluster [DBG] pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:17.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:16.902443+0000 mgr.x (mgr.34616) 168 : cluster [DBG] pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:20.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:18.903255+0000 mgr.x (mgr.34616) 169 : cluster [DBG] pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:18.903255+0000 mgr.x (mgr.34616) 169 : cluster [DBG] pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:18.903255+0000 mgr.x (mgr.34616) 169 : cluster [DBG] pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:20.903651+0000 mgr.x (mgr.34616) 170 : cluster [DBG] pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:20.903651+0000 mgr.x (mgr.34616) 170 : cluster [DBG] pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:20.903651+0000 mgr.x (mgr.34616) 170 : cluster [DBG] pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:24.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:22.904459+0000 mgr.x (mgr.34616) 171 : cluster [DBG] pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:22.904459+0000 mgr.x (mgr.34616) 171 : cluster [DBG] pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:22.904459+0000 mgr.x (mgr.34616) 171 : cluster [DBG] pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:25.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:24.904986+0000 mgr.x (mgr.34616) 172 : cluster [DBG] pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:25.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:24.904986+0000 mgr.x (mgr.34616) 172 : cluster [DBG] pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:25.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:24.904986+0000 mgr.x (mgr.34616) 172 : cluster [DBG] pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:26.905557+0000 mgr.x (mgr.34616) 173 : cluster [DBG] pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:27.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:26.905557+0000 mgr.x (mgr.34616) 173 : cluster [DBG] pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:27.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:26.905557+0000 mgr.x (mgr.34616) 173 : cluster [DBG] pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:28.906397+0000 mgr.x (mgr.34616) 174 : cluster [DBG] pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:29.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:28.906397+0000 mgr.x (mgr.34616) 174 : cluster [DBG] pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:29.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:28.906397+0000 mgr.x (mgr.34616) 174 : cluster [DBG] pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:30.906910+0000 mgr.x (mgr.34616) 175 : cluster [DBG] pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:31.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:30.906910+0000 mgr.x (mgr.34616) 175 : cluster [DBG] pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:31.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:30.906910+0000 mgr.x (mgr.34616) 175 : cluster [DBG] pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:32.907802+0000 mgr.x (mgr.34616) 176 : cluster [DBG] pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:32.907802+0000 mgr.x (mgr.34616) 176 : cluster [DBG] pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:32.907802+0000 mgr.x (mgr.34616) 176 : cluster [DBG] pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:35.276 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:34.908334+0000 mgr.x (mgr.34616) 177 : cluster [DBG] pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:35.276 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:35.004579+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:00:35.298 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:34.908334+0000 mgr.x (mgr.34616) 177 : cluster [DBG] pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:35.298 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:35.004579+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:00:35.298 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:34.908334+0000 mgr.x (mgr.34616) 177 : cluster [DBG] pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:35.298 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:35.004579+0000 mon.b (mon.2) 87 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:00:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:36.908720+0000 mgr.x (mgr.34616) 178 : cluster [DBG] pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:37.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:36.908720+0000 mgr.x (mgr.34616) 178 : cluster [DBG] pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:37.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:36.908720+0000 mgr.x (mgr.34616) 178 : cluster [DBG] pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:39.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:38.909469+0000 mgr.x (mgr.34616) 179 : cluster [DBG] pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:38.909469+0000 mgr.x (mgr.34616) 179 : cluster [DBG] pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:38.909469+0000 mgr.x (mgr.34616) 179 : cluster [DBG] pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:40.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:39.449067+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:40.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:39.456451+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:40.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:39.449067+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:40.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:39.456451+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:39.449067+0000 mon.a (mon.0) 736 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:40.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:39.456451+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:40.682041+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:40.687668+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:40.909880+0000 mgr.x (mgr.34616) 180 : cluster [DBG] pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:40.978043+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:00:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:40.978663+0000 mon.b (mon.2) 89 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:00:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:00:40.984631+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:40.682041+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:40.687668+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:40.909880+0000 mgr.x (mgr.34616) 180 : cluster [DBG] pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:42.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:40.978043+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:00:42.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:40.978663+0000 mon.b (mon.2) 89 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:00:42.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:00:40.984631+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:40.682041+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:40.687668+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:40.909880+0000 mgr.x (mgr.34616) 180 : cluster [DBG] pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:40.978043+0000 mon.b (mon.2) 88 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:40.978663+0000 mon.b (mon.2) 89 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:00:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:00:40.984631+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:00:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:00:40.979134+0000 mgr.x (mgr.34616) 181 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:00:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:40.979512+0000 mgr.x (mgr.34616) 182 : cluster [DBG] pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T09:00:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:40.979653+0000 mgr.x (mgr.34616) 183 : cluster [DBG] pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:00:40.979134+0000 mgr.x (mgr.34616) 181 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:00:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:40.979512+0000 mgr.x (mgr.34616) 182 : cluster [DBG] pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T09:00:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:40.979653+0000 mgr.x (mgr.34616) 183 : cluster [DBG] pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:00:40.979134+0000 mgr.x (mgr.34616) 181 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:00:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:40.979512+0000 mgr.x (mgr.34616) 182 : cluster [DBG] pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1016 B/s rd, 0 op/s 2024-09-17T09:00:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:40.979653+0000 mgr.x (mgr.34616) 183 : cluster [DBG] pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:45.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:42.980425+0000 mgr.x (mgr.34616) 184 : cluster [DBG] pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:45.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:42.980425+0000 mgr.x (mgr.34616) 184 : cluster [DBG] pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:42.980425+0000 mgr.x (mgr.34616) 184 : cluster [DBG] pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:44.980854+0000 mgr.x (mgr.34616) 185 : cluster [DBG] pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:47.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:44.980854+0000 mgr.x (mgr.34616) 185 : cluster [DBG] pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:44.980854+0000 mgr.x (mgr.34616) 185 : cluster [DBG] pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:49.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:46.981428+0000 mgr.x (mgr.34616) 186 : cluster [DBG] pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:46.981428+0000 mgr.x (mgr.34616) 186 : cluster [DBG] pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:46.981428+0000 mgr.x (mgr.34616) 186 : cluster [DBG] pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 634 B/s rd, 0 op/s 2024-09-17T09:00:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:48.982417+0000 mgr.x (mgr.34616) 187 : cluster [DBG] pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:48.982417+0000 mgr.x (mgr.34616) 187 : cluster [DBG] pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:48.982417+0000 mgr.x (mgr.34616) 187 : cluster [DBG] pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:50.983009+0000 mgr.x (mgr.34616) 188 : cluster [DBG] pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:00:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:50.983009+0000 mgr.x (mgr.34616) 188 : cluster [DBG] pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:00:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:50.983009+0000 mgr.x (mgr.34616) 188 : cluster [DBG] pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:00:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:52.983978+0000 mgr.x (mgr.34616) 189 : cluster [DBG] pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:52.983978+0000 mgr.x (mgr.34616) 189 : cluster [DBG] pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:52.983978+0000 mgr.x (mgr.34616) 189 : cluster [DBG] pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:00:57.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:54.984503+0000 mgr.x (mgr.34616) 190 : cluster [DBG] pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:57.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:54.984503+0000 mgr.x (mgr.34616) 190 : cluster [DBG] pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:57.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:54.984503+0000 mgr.x (mgr.34616) 190 : cluster [DBG] pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:59.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:00:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:56.985059+0000 mgr.x (mgr.34616) 191 : cluster [DBG] pgmap v154: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:00:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:56.985059+0000 mgr.x (mgr.34616) 191 : cluster [DBG] pgmap v154: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:00:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:00:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:56.985059+0000 mgr.x (mgr.34616) 191 : cluster [DBG] pgmap v154: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:00:58.985803+0000 mgr.x (mgr.34616) 192 : cluster [DBG] pgmap v155: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:00:58.985803+0000 mgr.x (mgr.34616) 192 : cluster [DBG] pgmap v155: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:00:58.985803+0000 mgr.x (mgr.34616) 192 : cluster [DBG] pgmap v155: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:03.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:00.986301+0000 mgr.x (mgr.34616) 193 : cluster [DBG] pgmap v156: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:00.986301+0000 mgr.x (mgr.34616) 193 : cluster [DBG] pgmap v156: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:00.986301+0000 mgr.x (mgr.34616) 193 : cluster [DBG] pgmap v156: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:02.987083+0000 mgr.x (mgr.34616) 194 : cluster [DBG] pgmap v157: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:05.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:02.987083+0000 mgr.x (mgr.34616) 194 : cluster [DBG] pgmap v157: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:05.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:02.987083+0000 mgr.x (mgr.34616) 194 : cluster [DBG] pgmap v157: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:04.987649+0000 mgr.x (mgr.34616) 195 : cluster [DBG] pgmap v158: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:07.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:04.987649+0000 mgr.x (mgr.34616) 195 : cluster [DBG] pgmap v158: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:04.987649+0000 mgr.x (mgr.34616) 195 : cluster [DBG] pgmap v158: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:06.988140+0000 mgr.x (mgr.34616) 196 : cluster [DBG] pgmap v159: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:06.988140+0000 mgr.x (mgr.34616) 196 : cluster [DBG] pgmap v159: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:06.988140+0000 mgr.x (mgr.34616) 196 : cluster [DBG] pgmap v159: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:08.988941+0000 mgr.x (mgr.34616) 197 : cluster [DBG] pgmap v160: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:08.988941+0000 mgr.x (mgr.34616) 197 : cluster [DBG] pgmap v160: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:11.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:08.988941+0000 mgr.x (mgr.34616) 197 : cluster [DBG] pgmap v160: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:13.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:10.989513+0000 mgr.x (mgr.34616) 198 : cluster [DBG] pgmap v161: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:10.989513+0000 mgr.x (mgr.34616) 198 : cluster [DBG] pgmap v161: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:10.989513+0000 mgr.x (mgr.34616) 198 : cluster [DBG] pgmap v161: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:15.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:12.990404+0000 mgr.x (mgr.34616) 199 : cluster [DBG] pgmap v162: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:12.990404+0000 mgr.x (mgr.34616) 199 : cluster [DBG] pgmap v162: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:12.990404+0000 mgr.x (mgr.34616) 199 : cluster [DBG] pgmap v162: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:17.060 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:14.990959+0000 mgr.x (mgr.34616) 200 : cluster [DBG] pgmap v163: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:17.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:14.990959+0000 mgr.x (mgr.34616) 200 : cluster [DBG] pgmap v163: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:17.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:14.990959+0000 mgr.x (mgr.34616) 200 : cluster [DBG] pgmap v163: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:16.991399+0000 mgr.x (mgr.34616) 201 : cluster [DBG] pgmap v164: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:16.991399+0000 mgr.x (mgr.34616) 201 : cluster [DBG] pgmap v164: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:16.991399+0000 mgr.x (mgr.34616) 201 : cluster [DBG] pgmap v164: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:18.992304+0000 mgr.x (mgr.34616) 202 : cluster [DBG] pgmap v165: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:18.992304+0000 mgr.x (mgr.34616) 202 : cluster [DBG] pgmap v165: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:18.992304+0000 mgr.x (mgr.34616) 202 : cluster [DBG] pgmap v165: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:20.992761+0000 mgr.x (mgr.34616) 203 : cluster [DBG] pgmap v166: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:20.992761+0000 mgr.x (mgr.34616) 203 : cluster [DBG] pgmap v166: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:20.992761+0000 mgr.x (mgr.34616) 203 : cluster [DBG] pgmap v166: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:25.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:22.993611+0000 mgr.x (mgr.34616) 204 : cluster [DBG] pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:22.993611+0000 mgr.x (mgr.34616) 204 : cluster [DBG] pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:22.993611+0000 mgr.x (mgr.34616) 204 : cluster [DBG] pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:24.994056+0000 mgr.x (mgr.34616) 205 : cluster [DBG] pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:24.994056+0000 mgr.x (mgr.34616) 205 : cluster [DBG] pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:24.994056+0000 mgr.x (mgr.34616) 205 : cluster [DBG] pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:26.994519+0000 mgr.x (mgr.34616) 206 : cluster [DBG] pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:26.994519+0000 mgr.x (mgr.34616) 206 : cluster [DBG] pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:26.994519+0000 mgr.x (mgr.34616) 206 : cluster [DBG] pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:28.995352+0000 mgr.x (mgr.34616) 207 : cluster [DBG] pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:28.995352+0000 mgr.x (mgr.34616) 207 : cluster [DBG] pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:28.995352+0000 mgr.x (mgr.34616) 207 : cluster [DBG] pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:30.995713+0000 mgr.x (mgr.34616) 208 : cluster [DBG] pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:33.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:30.995713+0000 mgr.x (mgr.34616) 208 : cluster [DBG] pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:30.995713+0000 mgr.x (mgr.34616) 208 : cluster [DBG] pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:32.996560+0000 mgr.x (mgr.34616) 209 : cluster [DBG] pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:32.996560+0000 mgr.x (mgr.34616) 209 : cluster [DBG] pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:32.996560+0000 mgr.x (mgr.34616) 209 : cluster [DBG] pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:34.997088+0000 mgr.x (mgr.34616) 210 : cluster [DBG] pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:37.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:34.997088+0000 mgr.x (mgr.34616) 210 : cluster [DBG] pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:34.997088+0000 mgr.x (mgr.34616) 210 : cluster [DBG] pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:36.997583+0000 mgr.x (mgr.34616) 211 : cluster [DBG] pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:36.997583+0000 mgr.x (mgr.34616) 211 : cluster [DBG] pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:36.997583+0000 mgr.x (mgr.34616) 211 : cluster [DBG] pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:41.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:38.998359+0000 mgr.x (mgr.34616) 212 : cluster [DBG] pgmap v175: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:41.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:38.998359+0000 mgr.x (mgr.34616) 212 : cluster [DBG] pgmap v175: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:41.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:38.998359+0000 mgr.x (mgr.34616) 212 : cluster [DBG] pgmap v175: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:01:41.095464+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:01:41.438428+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:01:41.439106+0000 mon.b (mon.2) 92 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:01:41.444356+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:01:41.095464+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:01:41.438428+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:01:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:01:41.439106+0000 mon.b (mon.2) 92 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:01:42.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:01:41.444356+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:01:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:01:41.095464+0000 mon.b (mon.2) 90 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:01:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:01:41.438428+0000 mon.b (mon.2) 91 : audit [DBG] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:01:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:01:41.439106+0000 mon.b (mon.2) 92 : audit [INF] from='mgr.34616 172.21.15.105:0/3548112934' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:01:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:01:41.444356+0000 mon.a (mon.0) 741 : audit [INF] from='mgr.34616 ' entity='mgr.x' 2024-09-17T09:01:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:40.998773+0000 mgr.x (mgr.34616) 213 : cluster [DBG] pgmap v176: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:01:41.439645+0000 mgr.x (mgr.34616) 214 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:01:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:41.440063+0000 mgr.x (mgr.34616) 215 : cluster [DBG] pgmap v177: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-09-17T09:01:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:41.440219+0000 mgr.x (mgr.34616) 216 : cluster [DBG] pgmap v178: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-09-17T09:01:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:40.998773+0000 mgr.x (mgr.34616) 213 : cluster [DBG] pgmap v176: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:43.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:01:41.439645+0000 mgr.x (mgr.34616) 214 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:01:43.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:41.440063+0000 mgr.x (mgr.34616) 215 : cluster [DBG] pgmap v177: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-09-17T09:01:43.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:41.440219+0000 mgr.x (mgr.34616) 216 : cluster [DBG] pgmap v178: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-09-17T09:01:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:40.998773+0000 mgr.x (mgr.34616) 213 : cluster [DBG] pgmap v176: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:01:41.439645+0000 mgr.x (mgr.34616) 214 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:01:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:41.440063+0000 mgr.x (mgr.34616) 215 : cluster [DBG] pgmap v177: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 980 B/s rd, 0 op/s 2024-09-17T09:01:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:41.440219+0000 mgr.x (mgr.34616) 216 : cluster [DBG] pgmap v178: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 606 B/s rd, 0 op/s 2024-09-17T09:01:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:43.440983+0000 mgr.x (mgr.34616) 217 : cluster [DBG] pgmap v179: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:43.440983+0000 mgr.x (mgr.34616) 217 : cluster [DBG] pgmap v179: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:43.440983+0000 mgr.x (mgr.34616) 217 : cluster [DBG] pgmap v179: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:45.441574+0000 mgr.x (mgr.34616) 218 : cluster [DBG] pgmap v180: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:45.441574+0000 mgr.x (mgr.34616) 218 : cluster [DBG] pgmap v180: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:45.441574+0000 mgr.x (mgr.34616) 218 : cluster [DBG] pgmap v180: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:49.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:47.442491+0000 mgr.x (mgr.34616) 219 : cluster [DBG] pgmap v181: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:47.442491+0000 mgr.x (mgr.34616) 219 : cluster [DBG] pgmap v181: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:47.442491+0000 mgr.x (mgr.34616) 219 : cluster [DBG] pgmap v181: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:51.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:49.442994+0000 mgr.x (mgr.34616) 220 : cluster [DBG] pgmap v182: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:49.442994+0000 mgr.x (mgr.34616) 220 : cluster [DBG] pgmap v182: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:49.442994+0000 mgr.x (mgr.34616) 220 : cluster [DBG] pgmap v182: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:51.443349+0000 mgr.x (mgr.34616) 221 : cluster [DBG] pgmap v183: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:01:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:51.443349+0000 mgr.x (mgr.34616) 221 : cluster [DBG] pgmap v183: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:01:53.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:51.443349+0000 mgr.x (mgr.34616) 221 : cluster [DBG] pgmap v183: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:01:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:53.444319+0000 mgr.x (mgr.34616) 222 : cluster [DBG] pgmap v184: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:55.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:53.444319+0000 mgr.x (mgr.34616) 222 : cluster [DBG] pgmap v184: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:53.444319+0000 mgr.x (mgr.34616) 222 : cluster [DBG] pgmap v184: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:55.444703+0000 mgr.x (mgr.34616) 223 : cluster [DBG] pgmap v185: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:57.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:55.444703+0000 mgr.x (mgr.34616) 223 : cluster [DBG] pgmap v185: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:55.444703+0000 mgr.x (mgr.34616) 223 : cluster [DBG] pgmap v185: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:01:59.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:01:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:57.445671+0000 mgr.x (mgr.34616) 224 : cluster [DBG] pgmap v186: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:01:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:57.445671+0000 mgr.x (mgr.34616) 224 : cluster [DBG] pgmap v186: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:01:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:01:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:57.445671+0000 mgr.x (mgr.34616) 224 : cluster [DBG] pgmap v186: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:01.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:01:59.446102+0000 mgr.x (mgr.34616) 225 : cluster [DBG] pgmap v187: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:01:59.446102+0000 mgr.x (mgr.34616) 225 : cluster [DBG] pgmap v187: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:01.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:01:59.446102+0000 mgr.x (mgr.34616) 225 : cluster [DBG] pgmap v187: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:01.446679+0000 mgr.x (mgr.34616) 226 : cluster [DBG] pgmap v188: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:01.446679+0000 mgr.x (mgr.34616) 226 : cluster [DBG] pgmap v188: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:01.446679+0000 mgr.x (mgr.34616) 226 : cluster [DBG] pgmap v188: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:03.447605+0000 mgr.x (mgr.34616) 227 : cluster [DBG] pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:03.447605+0000 mgr.x (mgr.34616) 227 : cluster [DBG] pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:03.447605+0000 mgr.x (mgr.34616) 227 : cluster [DBG] pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:05.448089+0000 mgr.x (mgr.34616) 228 : cluster [DBG] pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:05.448089+0000 mgr.x (mgr.34616) 228 : cluster [DBG] pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:05.448089+0000 mgr.x (mgr.34616) 228 : cluster [DBG] pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:07.448956+0000 mgr.x (mgr.34616) 229 : cluster [DBG] pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:07.448956+0000 mgr.x (mgr.34616) 229 : cluster [DBG] pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:07.448956+0000 mgr.x (mgr.34616) 229 : cluster [DBG] pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:09.449386+0000 mgr.x (mgr.34616) 230 : cluster [DBG] pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:09.449386+0000 mgr.x (mgr.34616) 230 : cluster [DBG] pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:09.449386+0000 mgr.x (mgr.34616) 230 : cluster [DBG] pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:13.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:11.449768+0000 mgr.x (mgr.34616) 231 : cluster [DBG] pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:11.449768+0000 mgr.x (mgr.34616) 231 : cluster [DBG] pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:11.449768+0000 mgr.x (mgr.34616) 231 : cluster [DBG] pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:15.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:13.450587+0000 mgr.x (mgr.34616) 232 : cluster [DBG] pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:13.450587+0000 mgr.x (mgr.34616) 232 : cluster [DBG] pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:13.450587+0000 mgr.x (mgr.34616) 232 : cluster [DBG] pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:17.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:15.451121+0000 mgr.x (mgr.34616) 233 : cluster [DBG] pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:15.451121+0000 mgr.x (mgr.34616) 233 : cluster [DBG] pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:17.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:15.451121+0000 mgr.x (mgr.34616) 233 : cluster [DBG] pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:17.451867+0000 mgr.x (mgr.34616) 234 : cluster [DBG] pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:17.451867+0000 mgr.x (mgr.34616) 234 : cluster [DBG] pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:17.451867+0000 mgr.x (mgr.34616) 234 : cluster [DBG] pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:21.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:19.452370+0000 mgr.x (mgr.34616) 235 : cluster [DBG] pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:19.452370+0000 mgr.x (mgr.34616) 235 : cluster [DBG] pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:19.452370+0000 mgr.x (mgr.34616) 235 : cluster [DBG] pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:23.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:21.452859+0000 mgr.x (mgr.34616) 236 : cluster [DBG] pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:23.207 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:21.452859+0000 mgr.x (mgr.34616) 236 : cluster [DBG] pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:21.452859+0000 mgr.x (mgr.34616) 236 : cluster [DBG] pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:23.621 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:02:25.025 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:23.453775+0000 mgr.x (mgr.34616) 237 : cluster [DBG] pgmap v199: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:25.025 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:23.453775+0000 mgr.x (mgr.34616) 237 : cluster [DBG] pgmap v199: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:25.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:23.453775+0000 mgr.x (mgr.34616) 237 : cluster [DBG] pgmap v199: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:25.757 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 106s ago 13m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (5m) 105s ago 13m 34.6M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (5m) 105s ago 12m 43.3M - 3.4 24ecd6d5f14c 9223e1c260cd 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443 running (8m) 105s ago 17m 485M - 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (2m) 106s ago 18m 436M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (18m) 106s ago 19m 64.9M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (17m) 105s ago 17m 60.8M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (17m) 106s ago 17m 55.8M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 106s ago 14m 15.0M - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (5m) 105s ago 14m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (17m) 106s ago 17m 48.0M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:02:25.758 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (16m) 106s ago 16m 48.8M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (16m) 106s ago 16m 44.9M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (16m) 106s ago 16m 47.8M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (15m) 105s ago 15m 50.5M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (15m) 105s ago 15m 45.3M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (15m) 105s ago 15m 47.2M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (15m) 105s ago 15m 51.2M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 error 105s ago 14m - - 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (13m) 106s ago 13m 38.3M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:02:25.759 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (13m) 105s ago 13m 38.7M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:02:26.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T09:02:27.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:25.454304+0000 mgr.x (mgr.34616) 238 : cluster [DBG] pgmap v200: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:27.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:25.748029+0000 mgr.x (mgr.34616) 239 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:02:27.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:25.454304+0000 mgr.x (mgr.34616) 238 : cluster [DBG] pgmap v200: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:27.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:25.748029+0000 mgr.x (mgr.34616) 239 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:02:27.513 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:25.454304+0000 mgr.x (mgr.34616) 238 : cluster [DBG] pgmap v200: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:27.513 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:25.748029+0000 mgr.x (mgr.34616) 239 : audit [DBG] from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:02:28.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:02:28.511 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:02:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:27.455270+0000 mgr.x (mgr.34616) 240 : cluster [DBG] pgmap v201: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:28.509382+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.27:0/3758878794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:02:29.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:27.455270+0000 mgr.x (mgr.34616) 240 : cluster [DBG] pgmap v201: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:29.068 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:28.509382+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.27:0/3758878794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:02:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:27.455270+0000 mgr.x (mgr.34616) 240 : cluster [DBG] pgmap v201: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:29.068 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:28.509382+0000 mon.c (mon.1) 31 : audit [DBG] from='client.? 172.21.15.27:0/3758878794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:02:29.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:02:31.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:29.455675+0000 mgr.x (mgr.34616) 241 : cluster [DBG] pgmap v202: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:31.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:29.455675+0000 mgr.x (mgr.34616) 241 : cluster [DBG] pgmap v202: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:29.455675+0000 mgr.x (mgr.34616) 241 : cluster [DBG] pgmap v202: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:31.134 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:02:31.134 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:02:31.134 INFO:teuthology.orchestra.run.smithi027.stdout: daemon prometheus.a on smithi105 is in error state 2024-09-17T09:02:31.787 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph -s' 2024-09-17T09:02:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:31.134044+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.27:0/1617101439' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:02:32.085 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:31.134044+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.27:0/1617101439' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:02:32.085 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:31.134044+0000 mon.a (mon.0) 742 : audit [DBG] from='client.? 172.21.15.27:0/1617101439' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:02:33.108 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:31.456186+0000 mgr.x (mgr.34616) 242 : cluster [DBG] pgmap v203: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:33.109 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:31.456186+0000 mgr.x (mgr.34616) 242 : cluster [DBG] pgmap v203: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:31.456186+0000 mgr.x (mgr.34616) 242 : cluster [DBG] pgmap v203: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: id: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_WARN 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: 1 failed cephadm daemon(s) 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:02:33.910 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 17m) 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: x(active, since 6m), standbys: y 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 333 objects, 604 KiB 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: io: 2024-09-17T09:02:33.911 INFO:teuthology.orchestra.run.smithi027.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-09-17T09:02:33.912 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:02:34.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph mgr fail' 2024-09-17T09:02:35.035 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:33.457102+0000 mgr.x (mgr.34616) 243 : cluster [DBG] pgmap v204: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:35.036 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:33.909569+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.27:0/4065133723' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:02:35.036 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:33.457102+0000 mgr.x (mgr.34616) 243 : cluster [DBG] pgmap v204: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:35.036 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:33.909569+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.27:0/4065133723' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:02:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:33.457102+0000 mgr.x (mgr.34616) 243 : cluster [DBG] pgmap v204: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:02:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:33.909569+0000 mon.a (mon.0) 743 : audit [DBG] from='client.? 172.21.15.27:0/4065133723' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:02:37.131 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:35.457575+0000 mgr.x (mgr.34616) 244 : cluster [DBG] pgmap v205: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:37.132 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.549384+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:02:37.132 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:36.558950+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-17T09:02:37.132 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:35.457575+0000 mgr.x (mgr.34616) 244 : cluster [DBG] pgmap v205: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:37.132 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.549384+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:02:37.132 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:36.558950+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-17T09:02:37.157 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:35.457575+0000 mgr.x (mgr.34616) 244 : cluster [DBG] pgmap v205: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:02:37.157 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.549384+0000 mon.a (mon.0) 744 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:02:37.157 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:36.558950+0000 mon.a (mon.0) 745 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-17T09:02:37.157 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:36.795+0000 7fbca7fd9640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T09:02:37.157 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ignoring --setuser ceph since I am not root 2024-09-17T09:02:37.158 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ignoring --setgroup ceph since I am not root 2024-09-17T09:02:37.158 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:36.845+0000 7f6f6fa20640 1 -- 172.21.15.105:0/1595336956 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555df5e74d00 con 0x555df5e49000 2024-09-17T09:02:37.158 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:36.947+0000 7f6f7227b040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:02:37.158 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.007+0000 7f6f7227b040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:02:37.158 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.156+0000 7f6f7227b040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:02:37.507 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.222+0000 7f6f7227b040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:02:37.507 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.369+0000 7f6f7227b040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:02:37.508 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.506+0000 7f6f7227b040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:02:37.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T09:02:38.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.795093+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:02:38.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:36.795199+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: y(active, starting, since 0.245154s) 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.796068+0000 mon.c (mon.1) 32 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.796213+0000 mon.c (mon.1) 33 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.796482+0000 mon.c (mon.1) 34 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.797169+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.797474+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.797806+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.798125+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.798513+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.798819+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.799140+0000 mon.c (mon.1) 41 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.799400+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.799666+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:02:38.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.799939+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.795093+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:36.795199+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: y(active, starting, since 0.245154s) 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.796068+0000 mon.c (mon.1) 32 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.796213+0000 mon.c (mon.1) 33 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.796482+0000 mon.c (mon.1) 34 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.797169+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.797474+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:02:38.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.797806+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.798125+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.798513+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.798819+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.799140+0000 mon.c (mon.1) 41 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.799400+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.799666+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.799939+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.800105+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:36.800577+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:37.264559+0000 mon.a (mon.0) 748 : cluster [INF] Manager daemon y is now available 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:37.273528+0000 mgr.y (mgr.14886) 1 : cephadm [INF] inventory: adjusted host smithi105 addr 'smithi105' -> '172.21.15.105' 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.276885+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:38.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.285473+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.312266+0000 mon.c (mon.1) 48 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.312691+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.327617+0000 mon.c (mon.1) 49 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:37.328039+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.800105+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:36.800577+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:37.264559+0000 mon.a (mon.0) 748 : cluster [INF] Manager daemon y is now available 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:37.273528+0000 mgr.y (mgr.14886) 1 : cephadm [INF] inventory: adjusted host smithi105 addr 'smithi105' -> '172.21.15.105' 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.276885+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:38.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.285473+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.312266+0000 mon.c (mon.1) 48 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.312691+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.327617+0000 mon.c (mon.1) 49 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.013 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:37.328039+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.109 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.834+0000 7f6f7227b040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:02:38.109 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:37.903+0000 7f6f7227b040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:02:38.109 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:38.107+0000 7f6f7227b040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:02:38.109 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.795093+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.27:0/2325626216' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:36.795199+0000 mon.a (mon.0) 747 : cluster [DBG] mgrmap e22: y(active, starting, since 0.245154s) 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.796068+0000 mon.c (mon.1) 32 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.796213+0000 mon.c (mon.1) 33 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.796482+0000 mon.c (mon.1) 34 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.797169+0000 mon.c (mon.1) 35 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.797474+0000 mon.c (mon.1) 36 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.797806+0000 mon.c (mon.1) 37 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.798125+0000 mon.c (mon.1) 38 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.798513+0000 mon.c (mon.1) 39 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.798819+0000 mon.c (mon.1) 40 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.799140+0000 mon.c (mon.1) 41 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:02:38.110 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.799400+0000 mon.c (mon.1) 42 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.799666+0000 mon.c (mon.1) 43 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.799939+0000 mon.c (mon.1) 44 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.800105+0000 mon.c (mon.1) 45 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:36.800577+0000 mon.c (mon.1) 46 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:37.264559+0000 mon.a (mon.0) 748 : cluster [INF] Manager daemon y is now available 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:37.273528+0000 mgr.y (mgr.14886) 1 : cephadm [INF] inventory: adjusted host smithi105 addr 'smithi105' -> '172.21.15.105' 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.276885+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.285473+0000 mon.c (mon.1) 47 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.312266+0000 mon.c (mon.1) 48 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.312691+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.327617+0000 mon.c (mon.1) 49 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:37.328039+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:02:38.914 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:37.802945+0000 mon.a (mon.0) 752 : cluster [DBG] mgrmap e23: y(active, since 1.2529s) 2024-09-17T09:02:38.915 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:37.823086+0000 mgr.y (mgr.14886) 2 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:38.915 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:37.802945+0000 mon.a (mon.0) 752 : cluster [DBG] mgrmap e23: y(active, since 1.2529s) 2024-09-17T09:02:38.915 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:37.823086+0000 mgr.y (mgr.14886) 2 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:39.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:37.802945+0000 mon.a (mon.0) 752 : cluster [DBG] mgrmap e23: y(active, since 1.2529s) 2024-09-17T09:02:39.081 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:37.823086+0000 mgr.y (mgr.14886) 2 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:39.081 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:39.015+0000 7f6f7227b040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:02:39.081 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:39.080+0000 7f6f7227b040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:02:39.979 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:38.799009+0000 mgr.y (mgr.14886) 3 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:38.800250+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:38.800275+0000 mon.a (mon.0) 754 : cluster [INF] Cluster is now healthy 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:38.810417+0000 mon.a (mon.0) 755 : cluster [DBG] mgrmap e24: y(active, since 2s) 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:38.799009+0000 mgr.y (mgr.14886) 3 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:38.800250+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:02:39.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:38.800275+0000 mon.a (mon.0) 754 : cluster [INF] Cluster is now healthy 2024-09-17T09:02:39.981 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:38.810417+0000 mon.a (mon.0) 755 : cluster [DBG] mgrmap e24: y(active, since 2s) 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:39.871+0000 7f6f7227b040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:39.998+0000 7f6f7227b040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:38.799009+0000 mgr.y (mgr.14886) 3 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:38.800250+0000 mon.a (mon.0) 753 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:38.800275+0000 mon.a (mon.0) 754 : cluster [INF] Cluster is now healthy 2024-09-17T09:02:40.152 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:38.810417+0000 mon.a (mon.0) 755 : cluster [DBG] mgrmap e24: y(active, since 2s) 2024-09-17T09:02:40.462 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.151+0000 7f6f7227b040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:02:40.462 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.223+0000 7f6f7227b040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:02:40.462 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.381+0000 7f6f7227b040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:02:40.462 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.461+0000 7f6f7227b040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:02:40.667 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:02:40.348+0000 7fccd3c9b640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:09:02:40] ENGINE Error in HTTPServer.serve 2024-09-17T09:02:40.667 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: Traceback (most recent call last): 2024-09-17T09:02:40.667 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._connections.run(self.expiration_interval) 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._run(expiration_interval) 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s = self.context.wrap_socket( 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: return self.sslsocket_class._create( 2024-09-17T09:02:40.668 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:02:40.669 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self.do_handshake() 2024-09-17T09:02:40.669 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:02:40.669 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._sslobj.do_handshake() 2024-09-17T09:02:40.669 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:02:40.669 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:02:40.773 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.538+0000 7f6f7227b040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:02:40.774 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.708+0000 7f6f7227b040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:02:40.774 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.772+0000 7f6f7227b040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:40.226287+0000 mgr.y (mgr.14886) 4 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTING 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:40.346673+0000 mgr.y (mgr.14886) 5 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:40.349261+0000 mgr.y (mgr.14886) 6 : cephadm [ERR] [17/Sep/2024:09:02:40] ENGINE Error in HTTPServer.serve 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Traceback (most recent call last): 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:02:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._connections.run(self.expiration_interval) 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._run(expiration_interval) 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s = self.context.wrap_socket( 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: return self.sslsocket_class._create( 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:02:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self.do_handshake() 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._sslobj.do_handshake() 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:40.447879+0000 mgr.y (mgr.14886) 7 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:40.447966+0000 mgr.y (mgr.14886) 8 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTED 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:40.226287+0000 mgr.y (mgr.14886) 4 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTING 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:40.346673+0000 mgr.y (mgr.14886) 5 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:40.349261+0000 mgr.y (mgr.14886) 6 : cephadm [ERR] [17/Sep/2024:09:02:40] ENGINE Error in HTTPServer.serve 2024-09-17T09:02:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: Traceback (most recent call last): 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._connections.run(self.expiration_interval) 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._run(expiration_interval) 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s = self.context.wrap_socket( 2024-09-17T09:02:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: return self.sslsocket_class._create( 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self.do_handshake() 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._sslobj.do_handshake() 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:40.447879+0000 mgr.y (mgr.14886) 7 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:02:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:40.447966+0000 mgr.y (mgr.14886) 8 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTED 2024-09-17T09:02:41.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.843+0000 7f6f7227b040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:02:41.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.927+0000 7f6f7227b040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:02:41.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:40.988+0000 7f6f7227b040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:02:41.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:40.226287+0000 mgr.y (mgr.14886) 4 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTING 2024-09-17T09:02:41.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:40.346673+0000 mgr.y (mgr.14886) 5 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:40.349261+0000 mgr.y (mgr.14886) 6 : cephadm [ERR] [17/Sep/2024:09:02:40] ENGINE Error in HTTPServer.serve 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Traceback (most recent call last): 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._connections.run(self.expiration_interval) 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._run(expiration_interval) 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:02:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s = self.context.wrap_socket( 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: return self.sslsocket_class._create( 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self.do_handshake() 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._sslobj.do_handshake() 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:40.447879+0000 mgr.y (mgr.14886) 7 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:40.447966+0000 mgr.y (mgr.14886) 8 : cephadm [INF] [17/Sep/2024:09:02:40] ENGINE Bus STARTED 2024-09-17T09:02:41.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:41.046+0000 7f6f7227b040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:02:41.943 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:02:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:02:41.855+0000 7f6f7227b040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:02:41.943 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:40.799603+0000 mgr.y (mgr.14886) 9 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:41.943 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:40.814883+0000 mon.a (mon.0) 756 : cluster [DBG] mgrmap e25: y(active, since 4s) 2024-09-17T09:02:42.169 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:40.799603+0000 mgr.y (mgr.14886) 9 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:42.170 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:40.814883+0000 mon.a (mon.0) 756 : cluster [DBG] mgrmap e25: y(active, since 4s) 2024-09-17T09:02:42.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:40.799603+0000 mgr.y (mgr.14886) 9 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:42.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:40.814883+0000 mon.a (mon.0) 756 : cluster [DBG] mgrmap e25: y(active, since 4s) 2024-09-17T09:02:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:41.858713+0000 mon.a (mon.0) 757 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:02:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:41.859293+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:02:43.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:41.859907+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:02:43.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:41.860568+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:02:43.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:41.860791+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:41.858713+0000 mon.a (mon.0) 757 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:41.859293+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:41.859907+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:41.860568+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:41.860791+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:41.858713+0000 mon.a (mon.0) 757 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:02:43.175 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:41.859293+0000 mon.b (mon.2) 93 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:02:43.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:41.859907+0000 mon.b (mon.2) 94 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:02:43.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:41.860568+0000 mon.b (mon.2) 95 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:02:43.176 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:41.860791+0000 mon.b (mon.2) 96 : audit [DBG] from='mgr.? 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:42.799942+0000 mgr.y (mgr.14886) 10 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:42.826088+0000 mon.a (mon.0) 758 : cluster [DBG] mgrmap e26: y(active, since 6s), standbys: x 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:42.829662+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:43.551032+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:43.560434+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:43.768420+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:43.773800+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:42.799942+0000 mgr.y (mgr.14886) 10 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:42.826088+0000 mon.a (mon.0) 758 : cluster [DBG] mgrmap e26: y(active, since 6s), standbys: x 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:42.829662+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:43.551032+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:43.560434+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:43.768420+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:43.773800+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:42.799942+0000 mgr.y (mgr.14886) 10 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:42.826088+0000 mon.a (mon.0) 758 : cluster [DBG] mgrmap e26: y(active, since 6s), standbys: x 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:42.829662+0000 mon.c (mon.1) 50 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:43.551032+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:43.560434+0000 mon.a (mon.0) 760 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:43.768420+0000 mon.a (mon.0) 761 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:44.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:43.773800+0000 mon.a (mon.0) 762 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.103357+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.113298+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.288260+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.297961+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.301633+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:45.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.302283+0000 mon.c (mon.1) 52 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:02:45.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:44.302869+0000 mgr.y (mgr.14886) 11 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:02:45.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:44.303677+0000 mgr.y (mgr.14886) 12 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-17T09:02:45.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:44.309190+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:44.378751+0000 mgr.y (mgr.14886) 13 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:02:45.226 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:44.632034+0000 mgr.y (mgr.14886) 14 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:02:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.103357+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.113298+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.288260+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.297961+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.301633+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.302283+0000 mon.c (mon.1) 52 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:44.302869+0000 mgr.y (mgr.14886) 11 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:44.303677+0000 mgr.y (mgr.14886) 12 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:44.309190+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:44.378751+0000 mgr.y (mgr.14886) 13 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:02:45.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:44.632034+0000 mgr.y (mgr.14886) 14 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.103357+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.113298+0000 mon.a (mon.0) 764 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.288260+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.297961+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.301633+0000 mon.c (mon.1) 51 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.302283+0000 mon.c (mon.1) 52 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:44.302869+0000 mgr.y (mgr.14886) 11 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:44.303677+0000 mgr.y (mgr.14886) 12 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:44.309190+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:44.378751+0000 mgr.y (mgr.14886) 13 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:02:45.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:44.632034+0000 mgr.y (mgr.14886) 14 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:02:46.400 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:45.296294+0000 mon.a (mon.0) 768 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:02:46.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:45.296294+0000 mon.a (mon.0) 768 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:02:46.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:45.296294+0000 mon.a (mon.0) 768 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:02:47.409 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:46.304231+0000 mgr.y (mgr.14886) 15 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T09:02:47.409 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:46.304231+0000 mgr.y (mgr.14886) 15 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T09:02:47.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:46.304231+0000 mgr.y (mgr.14886) 15 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-17T09:02:47.730 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:47 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:02:47.992 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:47 smithi105 podman[50108]: 2024-09-17 09:02:47.63357717 +0000 UTC m=+0.016875362 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:02:47.992 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:47 smithi105 podman[50108]: 2024-09-17 09:02:47.740520477 +0000 UTC m=+0.123818699 container create 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:48.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:47 smithi105 podman[50108]: 2024-09-17 09:02:47.992240007 +0000 UTC m=+0.375538193 container init 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:48.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:47 smithi105 podman[50108]: 2024-09-17 09:02:47.995542108 +0000 UTC m=+0.378840286 container start 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:48.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:02:48.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.025Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.027Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.027Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.028Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.029Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=7 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=7 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=7 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=7 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=7 2024-09-17T09:02:48.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=7 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=7 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.038Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=7 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.038Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=9.022821ms 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.039Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.039Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.039Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:02:48.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=info ts=2024-09-17T09:02:48.040Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[50123]: level=error ts=2024-09-17T09:02:48.040Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 bash[50108]: 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:02:48.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 podman[50140]: 2024-09-17 09:02:48.097926742 +0000 UTC m=+0.050146351 container died 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:49.019 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 podman[50140]: 2024-09-17 09:02:48.696698634 +0000 UTC m=+0.648918250 container remove 757355cab6754fcf5731cb9057d9a5cf6ed2f24a1e056fc72a49ec1de791d34d (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:49.019 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:02:49.019 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:48 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:02:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:48.113359+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:48.120282+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:48.121719+0000 mgr.y (mgr.14886) 16 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:02:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:48.122020+0000 mon.c (mon.1) 53 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:48.122343+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:48.126261+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:49.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:48.126991+0000 mgr.y (mgr.14886) 17 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:02:49.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:48.304857+0000 mgr.y (mgr.14886) 18 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:02:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:48.113359+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:48.120282+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:48.121719+0000 mgr.y (mgr.14886) 16 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:48.122020+0000 mon.c (mon.1) 53 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:48.122343+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:48.126261+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:48.126991+0000 mgr.y (mgr.14886) 17 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:48.304857+0000 mgr.y (mgr.14886) 18 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:48.113359+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:48.120282+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:48.121719+0000 mgr.y (mgr.14886) 16 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:02:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:48.122020+0000 mon.c (mon.1) 53 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:48.122343+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14886 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:02:49.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:48.126261+0000 mon.c (mon.1) 54 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:49.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:48.126991+0000 mgr.y (mgr.14886) 17 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:02:49.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:48.304857+0000 mgr.y (mgr.14886) 18 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:02:51.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:50.305345+0000 mgr.y (mgr.14886) 19 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:50.305345+0000 mgr.y (mgr.14886) 19 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:51.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:50.305345+0000 mgr.y (mgr.14886) 19 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:53.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:52.305852+0000 mgr.y (mgr.14886) 20 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:53.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.422403+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.429135+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.455 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.434622+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.435176+0000 mgr.y (mgr.14886) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.438920+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.439446+0000 mgr.y (mgr.14886) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.442929+0000 mon.c (mon.1) 57 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.443324+0000 mgr.y (mgr.14886) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.481803+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:52.959008+0000 mon.b (mon.2) 97 : audit [DBG] from='client.? 172.21.15.105:0/1930537918' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:02:53.456 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:53.305619+0000 mon.b (mon.2) 98 : audit [INF] from='client.? 172.21.15.105:0/4287276821' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:52.305852+0000 mgr.y (mgr.14886) 20 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.422403+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.429135+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.434622+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.435176+0000 mgr.y (mgr.14886) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.438920+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.439446+0000 mgr.y (mgr.14886) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.442929+0000 mon.c (mon.1) 57 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.443324+0000 mgr.y (mgr.14886) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.481803+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:52.959008+0000 mon.b (mon.2) 97 : audit [DBG] from='client.? 172.21.15.105:0/1930537918' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:53.305619+0000 mon.b (mon.2) 98 : audit [INF] from='client.? 172.21.15.105:0/4287276821' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:52.305852+0000 mgr.y (mgr.14886) 20 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.422403+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.429135+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.434622+0000 mon.c (mon.1) 55 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.435176+0000 mgr.y (mgr.14886) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.438920+0000 mon.c (mon.1) 56 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.439446+0000 mgr.y (mgr.14886) 22 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:02:53.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.442929+0000 mon.c (mon.1) 57 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.443324+0000 mgr.y (mgr.14886) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:02:53.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.481803+0000 mon.c (mon.1) 58 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:02:53.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:52.959008+0000 mon.b (mon.2) 97 : audit [DBG] from='client.? 172.21.15.105:0/1930537918' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:02:53.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:53.305619+0000 mon.b (mon.2) 98 : audit [INF] from='client.? 172.21.15.105:0/4287276821' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:02:55.654 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:54.306648+0000 mgr.y (mgr.14886) 24 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T09:02:55.757 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:54.306648+0000 mgr.y (mgr.14886) 24 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T09:02:55.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:54.306648+0000 mgr.y (mgr.14886) 24 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 11 op/s 2024-09-17T09:02:57.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:56.307054+0000 mgr.y (mgr.14886) 25 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:02:57.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:56.307054+0000 mgr.y (mgr.14886) 25 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:02:57.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:56.307054+0000 mgr.y (mgr.14886) 25 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:02:59.067 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:58 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 1. 2024-09-17T09:02:59.067 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:58 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:02:59.067 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:58 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:02:59.067 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:58 smithi105 podman[51297]: 2024-09-17 09:02:58.861534521 +0000 UTC m=+0.016471977 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:02:59.067 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:58 smithi105 podman[51297]: 2024-09-17 09:02:58.967836668 +0000 UTC m=+0.122774121 container create f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:59.524 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 podman[51297]: 2024-09-17 09:02:59.170443103 +0000 UTC m=+0.325380555 container init f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 podman[51297]: 2024-09-17 09:02:59.173195737 +0000 UTC m=+0.328133188 container start f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.203Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.204Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.204Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.207Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:02:59.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.207Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.215Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.215Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.215Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.216Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.216Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.217Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.217Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.217Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=8 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.217Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=10.280593ms 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.219Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.219Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:02:59.526 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.219Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=info ts=2024-09-17T09:02:59.220Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51311]: level=error ts=2024-09-17T09:02:59.220Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 bash[51297]: f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 2024-09-17T09:02:59.527 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:02:59.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 podman[51328]: 2024-09-17 09:02:59.262402852 +0000 UTC m=+0.034440519 container died f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:58.307888+0000 mgr.y (mgr.14886) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:58.716008+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:58.724320+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:58.725788+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:58.726564+0000 mon.c (mon.1) 60 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:02:59.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:02:58.727212+0000 mgr.y (mgr.14886) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:02:59.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:58.727762+0000 mgr.y (mgr.14886) 28 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 392 B/s rd, 0 op/s 2024-09-17T09:02:59.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:02:58.735252+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:02:59.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:02:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:58.765991+0000 mon.a (mon.0) 777 : cluster [WRN] Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:03:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:58.307888+0000 mgr.y (mgr.14886) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:03:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:58.716008+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:58.724320+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:58.725788+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:58.726564+0000 mon.c (mon.1) 60 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:02:58.727212+0000 mgr.y (mgr.14886) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:58.727762+0000 mgr.y (mgr.14886) 28 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 392 B/s rd, 0 op/s 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:02:58.735252+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:58.765991+0000 mon.a (mon.0) 777 : cluster [WRN] Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:58.307888+0000 mgr.y (mgr.14886) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:58.716008+0000 mon.a (mon.0) 774 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:58.724320+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:58.725788+0000 mon.c (mon.1) 59 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:03:00.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:58.726564+0000 mon.c (mon.1) 60 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:03:00.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:02:58.727212+0000 mgr.y (mgr.14886) 27 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:03:00.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:58.727762+0000 mgr.y (mgr.14886) 28 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 392 B/s rd, 0 op/s 2024-09-17T09:03:00.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:02:58.735252+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:03:00.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:02:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:58.765991+0000 mon.a (mon.0) 777 : cluster [WRN] Health check update: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:03:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 podman[51328]: 2024-09-17 09:02:59.838323585 +0000 UTC m=+0.610361253 container remove f47ed22ea4d188dd4a444fc6c1e43cde5ef6fb7bf4f3cb30f74313e71a07cf95 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:03:00.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:02:59 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:03:01.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:02:58.727987+0000 mgr.y (mgr.14886) 29 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:01.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:02:58.727987+0000 mgr.y (mgr.14886) 29 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:02:58.727987+0000 mgr.y (mgr.14886) 29 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:03.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:00.728576+0000 mgr.y (mgr.14886) 30 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:03.323 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:00.728576+0000 mgr.y (mgr.14886) 30 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:03.323 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:00.728576+0000 mgr.y (mgr.14886) 30 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 486 B/s rd, 0 op/s 2024-09-17T09:03:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:02.729001+0000 mgr.y (mgr.14886) 31 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:02.729001+0000 mgr.y (mgr.14886) 31 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:05.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:02.729001+0000 mgr.y (mgr.14886) 31 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:06.908 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:04.729712+0000 mgr.y (mgr.14886) 32 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:06.908 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:06.256916+0000 mon.b (mon.2) 99 : audit [DBG] from='client.? 172.21.15.105:0/4010786760' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:06.908 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:06.572676+0000 mon.b (mon.2) 100 : audit [INF] from='client.? 172.21.15.105:0/1801999480' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:04.729712+0000 mgr.y (mgr.14886) 32 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:06.256916+0000 mon.b (mon.2) 99 : audit [DBG] from='client.? 172.21.15.105:0/4010786760' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:06.572676+0000 mon.b (mon.2) 100 : audit [INF] from='client.? 172.21.15.105:0/1801999480' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:04.729712+0000 mgr.y (mgr.14886) 32 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:06.256916+0000 mon.b (mon.2) 99 : audit [DBG] from='client.? 172.21.15.105:0/4010786760' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:06.572676+0000 mon.b (mon.2) 100 : audit [INF] from='client.? 172.21.15.105:0/1801999480' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:06.730168+0000 mgr.y (mgr.14886) 33 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:06.730168+0000 mgr.y (mgr.14886) 33 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:09.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:06.730168+0000 mgr.y (mgr.14886) 33 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:10.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:08.730950+0000 mgr.y (mgr.14886) 34 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:03:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:08.730950+0000 mgr.y (mgr.14886) 34 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:03:10.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:08.730950+0000 mgr.y (mgr.14886) 34 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:03:10.300 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 2. 2024-09-17T09:03:10.300 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:10.300 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:03:10.300 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 podman[51592]: 2024-09-17 09:03:10.114991009 +0000 UTC m=+0.017616261 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:03:10.300 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 podman[51592]: 2024-09-17 09:03:10.214188412 +0000 UTC m=+0.116813655 container create ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 podman[51592]: 2024-09-17 09:03:10.424497166 +0000 UTC m=+0.327122407 container init ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 podman[51592]: 2024-09-17 09:03:10.427699486 +0000 UTC m=+0.330324726 container start ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.456Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.457Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.457Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:03:10.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.459Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.459Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.467Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=9 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.468Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=9.467216ms 2024-09-17T09:03:10.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.470Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.470Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.470Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.471Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.472Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.472Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.472Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=info ts=2024-09-17T09:03:10.472Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:03:10.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51607]: level=error ts=2024-09-17T09:03:10.472Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:03:10.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 bash[51592]: ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 2024-09-17T09:03:10.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:10.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:10 smithi105 podman[51624]: 2024-09-17 09:03:10.53036477 +0000 UTC m=+0.050057930 container died ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:11.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:11 smithi105 podman[51624]: 2024-09-17 09:03:11.069745436 +0000 UTC m=+0.589438585 container remove ea522ce12ada181b5ec47a9af7aa598ea50ee8bdae089d5568a2b10fe48d0747 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:11.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:11 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:03:11.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:11 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:03:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:10.731472+0000 mgr.y (mgr.14886) 35 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:10.731472+0000 mgr.y (mgr.14886) 35 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:10.731472+0000 mgr.y (mgr.14886) 35 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:12.732005+0000 mgr.y (mgr.14886) 36 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:14.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:12.732005+0000 mgr.y (mgr.14886) 36 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:12.732005+0000 mgr.y (mgr.14886) 36 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:16.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:14.732783+0000 mgr.y (mgr.14886) 37 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:16.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:14.732783+0000 mgr.y (mgr.14886) 37 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:14.732783+0000 mgr.y (mgr.14886) 37 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:16.733299+0000 mgr.y (mgr.14886) 38 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:18.060 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:16.733299+0000 mgr.y (mgr.14886) 38 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:18.060 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:16.733299+0000 mgr.y (mgr.14886) 38 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:20.148 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:18.733982+0000 mgr.y (mgr.14886) 39 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:20.148 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:19.467981+0000 mon.b (mon.2) 101 : audit [DBG] from='client.? 172.21.15.105:0/115173427' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:18.733982+0000 mgr.y (mgr.14886) 39 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:19.467981+0000 mon.b (mon.2) 101 : audit [DBG] from='client.? 172.21.15.105:0/115173427' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:18.733982+0000 mgr.y (mgr.14886) 39 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:19.467981+0000 mon.b (mon.2) 101 : audit [DBG] from='client.? 172.21.15.105:0/115173427' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:21.039 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:19.789958+0000 mon.b (mon.2) 102 : audit [INF] from='client.? 172.21.15.105:0/3003433227' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:19.789958+0000 mon.b (mon.2) 102 : audit [INF] from='client.? 172.21.15.105:0/3003433227' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:19.789958+0000 mon.b (mon.2) 102 : audit [INF] from='client.? 172.21.15.105:0/3003433227' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:21.429 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 3. 2024-09-17T09:03:21.429 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:21.429 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:03:21.695 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 podman[51894]: 2024-09-17 09:03:21.332487967 +0000 UTC m=+0.017700595 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:03:21.695 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 podman[51894]: 2024-09-17 09:03:21.453633961 +0000 UTC m=+0.138846620 container create 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:21.696 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 podman[51894]: 2024-09-17 09:03:21.662622037 +0000 UTC m=+0.347834650 container init 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:21.696 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 podman[51894]: 2024-09-17 09:03:21.665422593 +0000 UTC m=+0.350635207 container start 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:22.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:20.734607+0000 mgr.y (mgr.14886) 40 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.694Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.694Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.694Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.694Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.695Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.695Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.696Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.696Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.698Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.698Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=10 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.706Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=10 2024-09-17T09:03:22.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.706Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.706Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.707Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.708Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=10 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.708Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=9.829985ms 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.710Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:03:22.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.710Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.710Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=info ts=2024-09-17T09:03:21.711Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:03:22.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[51909]: level=error ts=2024-09-17T09:03:21.711Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:03:22.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 bash[51894]: 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 2024-09-17T09:03:22.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:22.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:21 smithi105 podman[51926]: 2024-09-17 09:03:21.75332097 +0000 UTC m=+0.034848848 container died 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:20.734607+0000 mgr.y (mgr.14886) 40 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:20.734607+0000 mgr.y (mgr.14886) 40 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:22.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:22 smithi105 podman[51926]: 2024-09-17 09:03:22.292802648 +0000 UTC m=+0.574330562 container remove 896296fa35376941b489b747bf3d9e8591b0f16849f35436a2f3672eef441517 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:22.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:22 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:03:22.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:22 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:03:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:22.735099+0000 mgr.y (mgr.14886) 41 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:24.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:22.735099+0000 mgr.y (mgr.14886) 41 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:22.735099+0000 mgr.y (mgr.14886) 41 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:26.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:24.735896+0000 mgr.y (mgr.14886) 42 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:24.735896+0000 mgr.y (mgr.14886) 42 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:24.735896+0000 mgr.y (mgr.14886) 42 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:26.736405+0000 mgr.y (mgr.14886) 43 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:26.736405+0000 mgr.y (mgr.14886) 43 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:26.736405+0000 mgr.y (mgr.14886) 43 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:30.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:28.737167+0000 mgr.y (mgr.14886) 44 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:28.737167+0000 mgr.y (mgr.14886) 44 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:28.737167+0000 mgr.y (mgr.14886) 44 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:31.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:30.737773+0000 mgr.y (mgr.14886) 45 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:30.737773+0000 mgr.y (mgr.14886) 45 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:30.737773+0000 mgr.y (mgr.14886) 45 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:32.835 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 4. 2024-09-17T09:03:32.835 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:32.835 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:03:32.835 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 podman[52112]: 2024-09-17 09:03:32.612929853 +0000 UTC m=+0.017144263 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:03:32.835 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 podman[52112]: 2024-09-17 09:03:32.726157117 +0000 UTC m=+0.130371523 container create 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:33.132 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 podman[52112]: 2024-09-17 09:03:32.928116018 +0000 UTC m=+0.332330426 container init 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:33.132 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 podman[52112]: 2024-09-17 09:03:32.931054342 +0000 UTC m=+0.335268741 container start 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:33.132 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.960Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.960Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.960Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.961Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.961Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.961Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.962Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.962Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.964Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.964Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=11 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.972Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=11 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=11 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=11 2024-09-17T09:03:33.133 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.973Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.974Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.974Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=11 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.974Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=10.203266ms 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.977Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.977Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.977Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:03:33.134 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=info ts=2024-09-17T09:03:32.978Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[52130]: level=error ts=2024-09-17T09:03:32.978Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 bash[52112]: 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:33.135 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 podman[52167]: 2024-09-17 09:03:33.010476567 +0000 UTC m=+0.024966540 container died 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:33.929 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 podman[52167]: 2024-09-17 09:03:33.574861958 +0000 UTC m=+0.589351967 container remove 1b20558e1bbaaf7fe0b6d94f75f59847880ea9abfba43719731bba8fdbf3696f (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:03:34.085 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:03:34.085 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:33 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:03:34.085 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:32.738286+0000 mgr.y (mgr.14886) 46 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:34.085 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:32.776088+0000 mon.b (mon.2) 103 : audit [DBG] from='client.? 172.21.15.105:0/2184635956' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:34.085 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:33.126977+0000 mon.b (mon.2) 104 : audit [INF] from='client.? 172.21.15.105:0/3802537930' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:32.738286+0000 mgr.y (mgr.14886) 46 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:32.776088+0000 mon.b (mon.2) 103 : audit [DBG] from='client.? 172.21.15.105:0/2184635956' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:33.126977+0000 mon.b (mon.2) 104 : audit [INF] from='client.? 172.21.15.105:0/3802537930' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:32.738286+0000 mgr.y (mgr.14886) 46 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:32.776088+0000 mon.b (mon.2) 103 : audit [DBG] from='client.? 172.21.15.105:0/2184635956' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:33.126977+0000 mon.b (mon.2) 104 : audit [INF] from='client.? 172.21.15.105:0/3802537930' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:34.739196+0000 mgr.y (mgr.14886) 47 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:03:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:34.739196+0000 mgr.y (mgr.14886) 47 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:03:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:34.739196+0000 mgr.y (mgr.14886) 47 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:03:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:36.739748+0000 mgr.y (mgr.14886) 48 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:36.739748+0000 mgr.y (mgr.14886) 48 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:36.739748+0000 mgr.y (mgr.14886) 48 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:40.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:38.740697+0000 mgr.y (mgr.14886) 49 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:38.740697+0000 mgr.y (mgr.14886) 49 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:38.740697+0000 mgr.y (mgr.14886) 49 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:40.741200+0000 mgr.y (mgr.14886) 50 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:40.741200+0000 mgr.y (mgr.14886) 50 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:40.741200+0000 mgr.y (mgr.14886) 50 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:42.741729+0000 mgr.y (mgr.14886) 51 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:42.741729+0000 mgr.y (mgr.14886) 51 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:42.741729+0000 mgr.y (mgr.14886) 51 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:44.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:43 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 5. 2024-09-17T09:03:44.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:43 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:44.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:43 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Start request repeated too quickly. 2024-09-17T09:03:44.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:43 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:03:44.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:03:43 smithi105 systemd[1]: Failed to start Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:03:46.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:44.742506+0000 mgr.y (mgr.14886) 52 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:44.742506+0000 mgr.y (mgr.14886) 52 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:44.742506+0000 mgr.y (mgr.14886) 52 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:47.119 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:46.156342+0000 mon.b (mon.2) 105 : audit [DBG] from='client.? 172.21.15.105:0/1462530140' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:47.119 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:46.472510+0000 mon.b (mon.2) 106 : audit [INF] from='client.? 172.21.15.105:0/1426233102' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:46.156342+0000 mon.b (mon.2) 105 : audit [DBG] from='client.? 172.21.15.105:0/1462530140' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:46.472510+0000 mon.b (mon.2) 106 : audit [INF] from='client.? 172.21.15.105:0/1426233102' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:46.156342+0000 mon.b (mon.2) 105 : audit [DBG] from='client.? 172.21.15.105:0/1462530140' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:03:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:46.472510+0000 mon.b (mon.2) 106 : audit [INF] from='client.? 172.21.15.105:0/1426233102' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: access denied 2024-09-17T09:03:48.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:46.743074+0000 mgr.y (mgr.14886) 53 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:48.159 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:46.743074+0000 mgr.y (mgr.14886) 53 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:46.743074+0000 mgr.y (mgr.14886) 53 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:03:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:48.743942+0000 mgr.y (mgr.14886) 54 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:48.743942+0000 mgr.y (mgr.14886) 54 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:48.743942+0000 mgr.y (mgr.14886) 54 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:52.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:50.744402+0000 mgr.y (mgr.14886) 55 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:52.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:50.744402+0000 mgr.y (mgr.14886) 55 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:50.744402+0000 mgr.y (mgr.14886) 55 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:52.745020+0000 mgr.y (mgr.14886) 56 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:52.745020+0000 mgr.y (mgr.14886) 56 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:52.745020+0000 mgr.y (mgr.14886) 56 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:56.080 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:54.746041+0000 mgr.y (mgr.14886) 57 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:54.746041+0000 mgr.y (mgr.14886) 57 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:54.746041+0000 mgr.y (mgr.14886) 57 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:56.746642+0000 mgr.y (mgr.14886) 58 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:56.746642+0000 mgr.y (mgr.14886) 58 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:03:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:56.746642+0000 mgr.y (mgr.14886) 58 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:04:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:03:58.747378+0000 mgr.y (mgr.14886) 59 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:04:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:03:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:03:58.847671+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:04:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:03:58.747378+0000 mgr.y (mgr.14886) 59 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:04:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:03:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:03:58.847671+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:04:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:03:58.747378+0000 mgr.y (mgr.14886) 59 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:04:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:03:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:03:58.847671+0000 mon.c (mon.1) 61 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:04:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:00.747994+0000 mgr.y (mgr.14886) 60 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:00.747994+0000 mgr.y (mgr.14886) 60 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:00.747994+0000 mgr.y (mgr.14886) 60 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:02.748680+0000 mgr.y (mgr.14886) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:04:03.773262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:04:03.784228+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:02.748680+0000 mgr.y (mgr.14886) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:04:03.773262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:04:03.784228+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:02.748680+0000 mgr.y (mgr.14886) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:04:03.773262+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:04:03.784228+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:04:04.085403+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:04:04.086393+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:04:04.087157+0000 mgr.y (mgr.14886) 62 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:04.088055+0000 mgr.y (mgr.14886) 63 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:04.088299+0000 mgr.y (mgr.14886) 64 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:04:04.095517+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:04:04.085403+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:04:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:04:04.086393+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:04:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:04:04.087157+0000 mgr.y (mgr.14886) 62 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:04:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:04.088055+0000 mgr.y (mgr.14886) 63 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:04.088299+0000 mgr.y (mgr.14886) 64 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:04:04.095517+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:04:04.085403+0000 mon.c (mon.1) 62 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:04:04.086393+0000 mon.c (mon.1) 63 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:04:04.087157+0000 mgr.y (mgr.14886) 62 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:04.088055+0000 mgr.y (mgr.14886) 63 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:04.088299+0000 mgr.y (mgr.14886) 64 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:04:04.095517+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:04:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:06.088887+0000 mgr.y (mgr.14886) 65 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:07.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:06.088887+0000 mgr.y (mgr.14886) 65 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:06.088887+0000 mgr.y (mgr.14886) 65 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:08.089674+0000 mgr.y (mgr.14886) 66 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:08.089674+0000 mgr.y (mgr.14886) 66 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:08.089674+0000 mgr.y (mgr.14886) 66 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:11.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:10.090088+0000 mgr.y (mgr.14886) 67 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:11.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:10.090088+0000 mgr.y (mgr.14886) 67 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:11.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:10.090088+0000 mgr.y (mgr.14886) 67 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:13.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:12.090535+0000 mgr.y (mgr.14886) 68 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:13.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:12.090535+0000 mgr.y (mgr.14886) 68 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:13.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:12.090535+0000 mgr.y (mgr.14886) 68 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:14.091307+0000 mgr.y (mgr.14886) 69 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:15.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:14.091307+0000 mgr.y (mgr.14886) 69 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:15.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:14.091307+0000 mgr.y (mgr.14886) 69 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:17.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:16.091652+0000 mgr.y (mgr.14886) 70 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:17.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:16.091652+0000 mgr.y (mgr.14886) 70 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:16.091652+0000 mgr.y (mgr.14886) 70 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:19.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:18.092361+0000 mgr.y (mgr.14886) 71 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:18.092361+0000 mgr.y (mgr.14886) 71 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:18.092361+0000 mgr.y (mgr.14886) 71 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:21.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:20.092804+0000 mgr.y (mgr.14886) 72 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:21.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:20.092804+0000 mgr.y (mgr.14886) 72 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:21.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:20.092804+0000 mgr.y (mgr.14886) 72 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:23.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:22.093391+0000 mgr.y (mgr.14886) 73 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:23.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:22.093391+0000 mgr.y (mgr.14886) 73 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:23.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:22.093391+0000 mgr.y (mgr.14886) 73 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:25.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:24.094294+0000 mgr.y (mgr.14886) 74 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:25.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:24.094294+0000 mgr.y (mgr.14886) 74 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:25.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:24.094294+0000 mgr.y (mgr.14886) 74 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:27.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:26.094751+0000 mgr.y (mgr.14886) 75 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:27.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:26.094751+0000 mgr.y (mgr.14886) 75 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:27.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:26.094751+0000 mgr.y (mgr.14886) 75 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:28.095566+0000 mgr.y (mgr.14886) 76 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:28.095566+0000 mgr.y (mgr.14886) 76 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:28.095566+0000 mgr.y (mgr.14886) 76 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:31.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:30.095996+0000 mgr.y (mgr.14886) 77 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:31.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:30.095996+0000 mgr.y (mgr.14886) 77 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:31.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:30.095996+0000 mgr.y (mgr.14886) 77 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:33.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:32.096535+0000 mgr.y (mgr.14886) 78 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:33.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:32.096535+0000 mgr.y (mgr.14886) 78 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:32.096535+0000 mgr.y (mgr.14886) 78 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:35.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:34.097296+0000 mgr.y (mgr.14886) 79 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:35.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:34.097296+0000 mgr.y (mgr.14886) 79 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:34.097296+0000 mgr.y (mgr.14886) 79 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:37.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:36.097721+0000 mgr.y (mgr.14886) 80 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:37.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:36.097721+0000 mgr.y (mgr.14886) 80 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:37.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:36.097721+0000 mgr.y (mgr.14886) 80 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:39.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:38.098481+0000 mgr.y (mgr.14886) 81 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:38.098481+0000 mgr.y (mgr.14886) 81 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:38.098481+0000 mgr.y (mgr.14886) 81 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:41.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:40.098982+0000 mgr.y (mgr.14886) 82 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:41.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:40.098982+0000 mgr.y (mgr.14886) 82 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:41.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:40.098982+0000 mgr.y (mgr.14886) 82 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:43.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:42.099450+0000 mgr.y (mgr.14886) 83 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:43.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:42.099450+0000 mgr.y (mgr.14886) 83 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:43.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:42.099450+0000 mgr.y (mgr.14886) 83 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:44.100322+0000 mgr.y (mgr.14886) 84 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:45.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:44.100322+0000 mgr.y (mgr.14886) 84 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:45.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:44.100322+0000 mgr.y (mgr.14886) 84 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:46.100816+0000 mgr.y (mgr.14886) 85 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:47.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:46.100816+0000 mgr.y (mgr.14886) 85 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:46.100816+0000 mgr.y (mgr.14886) 85 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:49.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:48.101544+0000 mgr.y (mgr.14886) 86 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:48.101544+0000 mgr.y (mgr.14886) 86 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:48.101544+0000 mgr.y (mgr.14886) 86 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:51.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:50.101994+0000 mgr.y (mgr.14886) 87 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:51.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:50.101994+0000 mgr.y (mgr.14886) 87 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:50.101994+0000 mgr.y (mgr.14886) 87 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:53.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:52.102445+0000 mgr.y (mgr.14886) 88 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:52.102445+0000 mgr.y (mgr.14886) 88 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:53.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:52.102445+0000 mgr.y (mgr.14886) 88 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:55.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:54.103139+0000 mgr.y (mgr.14886) 89 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:55.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:54.103139+0000 mgr.y (mgr.14886) 89 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:55.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:54.103139+0000 mgr.y (mgr.14886) 89 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:57.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:56.103536+0000 mgr.y (mgr.14886) 90 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:56.103536+0000 mgr.y (mgr.14886) 90 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:56.103536+0000 mgr.y (mgr.14886) 90 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:04:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:04:58.104400+0000 mgr.y (mgr.14886) 91 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:04:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:04:58.104400+0000 mgr.y (mgr.14886) 91 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:04:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:04:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:04:58.104400+0000 mgr.y (mgr.14886) 91 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:00.104782+0000 mgr.y (mgr.14886) 92 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:00.104782+0000 mgr.y (mgr.14886) 92 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:01.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:00.104782+0000 mgr.y (mgr.14886) 92 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:02.105223+0000 mgr.y (mgr.14886) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:02.105223+0000 mgr.y (mgr.14886) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:02.105223+0000 mgr.y (mgr.14886) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:04.106030+0000 mgr.y (mgr.14886) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:04.208910+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:04.572203+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:05:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:04.572935+0000 mon.c (mon.1) 66 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:05:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:04.573553+0000 mgr.y (mgr.14886) 95 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:05:05.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:04.573981+0000 mgr.y (mgr.14886) 96 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:04.574155+0000 mgr.y (mgr.14886) 97 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:04.580603+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:05:05.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:04.106030+0000 mgr.y (mgr.14886) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:04.208910+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:04.572203+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:04.572935+0000 mon.c (mon.1) 66 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:04.573553+0000 mgr.y (mgr.14886) 95 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:04.573981+0000 mgr.y (mgr.14886) 96 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:04.574155+0000 mgr.y (mgr.14886) 97 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:04.580603+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:04.106030+0000 mgr.y (mgr.14886) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:04.208910+0000 mon.c (mon.1) 64 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:04.572203+0000 mon.c (mon.1) 65 : audit [DBG] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:04.572935+0000 mon.c (mon.1) 66 : audit [INF] from='mgr.14886 172.21.15.27:0/1784654869' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:04.573553+0000 mgr.y (mgr.14886) 95 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:04.573981+0000 mgr.y (mgr.14886) 96 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:04.574155+0000 mgr.y (mgr.14886) 97 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:04.580603+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.14886 ' entity='mgr.y' 2024-09-17T09:05:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:06.574756+0000 mgr.y (mgr.14886) 98 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:06.574756+0000 mgr.y (mgr.14886) 98 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:08.012 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:06.574756+0000 mgr.y (mgr.14886) 98 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:09.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:08.575546+0000 mgr.y (mgr.14886) 99 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:08.575546+0000 mgr.y (mgr.14886) 99 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:08.575546+0000 mgr.y (mgr.14886) 99 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:10.575956+0000 mgr.y (mgr.14886) 100 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:10.575956+0000 mgr.y (mgr.14886) 100 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:12.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:10.575956+0000 mgr.y (mgr.14886) 100 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:12.576299+0000 mgr.y (mgr.14886) 101 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:12.576299+0000 mgr.y (mgr.14886) 101 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:12.576299+0000 mgr.y (mgr.14886) 101 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:14.577044+0000 mgr.y (mgr.14886) 102 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:16.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:14.577044+0000 mgr.y (mgr.14886) 102 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:16.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:14.577044+0000 mgr.y (mgr.14886) 102 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:16.577489+0000 mgr.y (mgr.14886) 103 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:16.577489+0000 mgr.y (mgr.14886) 103 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:16.577489+0000 mgr.y (mgr.14886) 103 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:18.578277+0000 mgr.y (mgr.14886) 104 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:18.578277+0000 mgr.y (mgr.14886) 104 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:18.578277+0000 mgr.y (mgr.14886) 104 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:21.961 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:20.578770+0000 mgr.y (mgr.14886) 105 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:21.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:20.578770+0000 mgr.y (mgr.14886) 105 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:22.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:20.578770+0000 mgr.y (mgr.14886) 105 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:22.579286+0000 mgr.y (mgr.14886) 106 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:22.579286+0000 mgr.y (mgr.14886) 106 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:22.579286+0000 mgr.y (mgr.14886) 106 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:26.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:24.580278+0000 mgr.y (mgr.14886) 107 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:26.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:24.580278+0000 mgr.y (mgr.14886) 107 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:24.580278+0000 mgr.y (mgr.14886) 107 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:26.580675+0000 mgr.y (mgr.14886) 108 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:28.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:26.580675+0000 mgr.y (mgr.14886) 108 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:28.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:26.580675+0000 mgr.y (mgr.14886) 108 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:29.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:28.581499+0000 mgr.y (mgr.14886) 109 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:28.581499+0000 mgr.y (mgr.14886) 109 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:28.581499+0000 mgr.y (mgr.14886) 109 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:32.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:30.581991+0000 mgr.y (mgr.14886) 110 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:30.581991+0000 mgr.y (mgr.14886) 110 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:30.581991+0000 mgr.y (mgr.14886) 110 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:34.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:32.582448+0000 mgr.y (mgr.14886) 111 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:32.582448+0000 mgr.y (mgr.14886) 111 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:32.582448+0000 mgr.y (mgr.14886) 111 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:34.583382+0000 mgr.y (mgr.14886) 112 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:36.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:34.583382+0000 mgr.y (mgr.14886) 112 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:34.583382+0000 mgr.y (mgr.14886) 112 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:38.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:36.583833+0000 mgr.y (mgr.14886) 113 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:36.583833+0000 mgr.y (mgr.14886) 113 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:38.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:36.583833+0000 mgr.y (mgr.14886) 113 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:38.584661+0000 mgr.y (mgr.14886) 114 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:38.584661+0000 mgr.y (mgr.14886) 114 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:38.584661+0000 mgr.y (mgr.14886) 114 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:39.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:05:41.987 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 2m ago 16m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (8m) 98s ago 16m 35.0M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 98s ago 16m - - 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443 running (11m) 98s ago 20m 435M - 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (6m) 2m ago 22m 459M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (22m) 2m ago 22m 65.2M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (21m) 98s ago 21m 59.6M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (21m) 2m ago 21m 46.4M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 17m 15.0M - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T09:05:41.988 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (8m) 98s ago 17m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (20m) 2m ago 20m 48.2M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (20m) 2m ago 20m 49.1M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (19m) 2m ago 19m 44.0M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (19m) 2m ago 19m 48.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (19m) 98s ago 19m 50.7M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (18m) 98s ago 18m 45.7M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (18m) 98s ago 18m 47.5M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (18m) 98s ago 18m 51.7M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 error 98s ago 17m - - 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (16m) 2m ago 16m 39.1M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:05:41.989 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (16m) 98s ago 16m 40.1M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:05:41.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:40.585023+0000 mgr.y (mgr.14886) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:41.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:40.585023+0000 mgr.y (mgr.14886) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:40.585023+0000 mgr.y (mgr.14886) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:42.574 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T09:05:43.922 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:41.979053+0000 mgr.y (mgr.14886) 116 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:05:43.923 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:42.585425+0000 mgr.y (mgr.14886) 117 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:43.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:41.979053+0000 mgr.y (mgr.14886) 116 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:05:43.923 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:42.585425+0000 mgr.y (mgr.14886) 117 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:41.979053+0000 mgr.y (mgr.14886) 116 : audit [DBG] from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:05:44.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:42.585425+0000 mgr.y (mgr.14886) 117 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:44.664 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:05:44.664 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:05:44.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:05:44.666 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:05:44.666 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:05:44.666 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:05:44.666 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:05:44.666 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:05:45.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph -s' 2024-09-17T09:05:45.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:44.586035+0000 mgr.y (mgr.14886) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:45.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:44.663960+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.27:0/4051888791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:45.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:44.586035+0000 mgr.y (mgr.14886) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:45.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:44.663960+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.27:0/4051888791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:44.586035+0000 mgr.y (mgr.14886) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:44.663960+0000 mon.a (mon.0) 782 : audit [DBG] from='client.? 172.21.15.27:0/4051888791' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:47.656 INFO:teuthology.orchestra.run.smithi027.stdout: cluster: 2024-09-17T09:05:47.656 INFO:teuthology.orchestra.run.smithi027.stdout: id: d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:05:47.656 INFO:teuthology.orchestra.run.smithi027.stdout: health: HEALTH_WARN 2024-09-17T09:05:47.656 INFO:teuthology.orchestra.run.smithi027.stdout: 2 failed cephadm daemon(s) 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: services: 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: mon: 3 daemons, quorum a,c,b (age 21m) 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: mgr: y(active, since 3m), standbys: x 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 18m) 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: data: 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: pools: 6 pools, 161 pgs 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: objects: 333 objects, 604 KiB 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: usage: 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: pgs: 161 active+clean 2024-09-17T09:05:47.657 INFO:teuthology.orchestra.run.smithi027.stdout: 2024-09-17T09:05:48.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:46.586556+0000 mgr.y (mgr.14886) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:48.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:46.586556+0000 mgr.y (mgr.14886) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:46.586556+0000 mgr.y (mgr.14886) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:48.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:05:48.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:47.655365+0000 mon.c (mon.1) 67 : audit [DBG] from='client.? 172.21.15.27:0/1972184928' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:05:48.919 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:47.655365+0000 mon.c (mon.1) 67 : audit [DBG] from='client.? 172.21.15.27:0/1972184928' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:05:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:47.655365+0000 mon.c (mon.1) 67 : audit [DBG] from='client.? 172.21.15.27:0/1972184928' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:05:49.731 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:48.587465+0000 mgr.y (mgr.14886) 120 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:49.732 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:48.587465+0000 mgr.y (mgr.14886) 120 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:48.587465+0000 mgr.y (mgr.14886) 120 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:50.242 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 2 failed cephadm daemon(s) 2024-09-17T09:05:50.243 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 2 failed cephadm daemon(s) 2024-09-17T09:05:50.243 INFO:teuthology.orchestra.run.smithi027.stdout: daemon prometheus.a on smithi105 is in error state 2024-09-17T09:05:50.243 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in error state 2024-09-17T09:05:50.812 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:50.242508+0000 mon.c (mon.1) 68 : audit [DBG] from='client.? 172.21.15.27:0/3384884722' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:05:50.813 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:50.242508+0000 mon.c (mon.1) 68 : audit [DBG] from='client.? 172.21.15.27:0/3384884722' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:05:50.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-17T09:05:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:50.242508+0000 mon.c (mon.1) 68 : audit [DBG] from='client.? 172.21.15.27:0/3384884722' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:05:51.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:50.587817+0000 mgr.y (mgr.14886) 121 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:51.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:50.587817+0000 mgr.y (mgr.14886) 121 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:50.587817+0000 mgr.y (mgr.14886) 121 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:52.844 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:05:53.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph mgr fail' 2024-09-17T09:05:53.851 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:52.588137+0000 mgr.y (mgr.14886) 122 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:53.851 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:52.832517+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.27:0/1141443849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:53.851 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:52.588137+0000 mgr.y (mgr.14886) 122 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:53.851 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:52.832517+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.27:0/1141443849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:52.588137+0000 mgr.y (mgr.14886) 122 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:52.832517+0000 mon.a (mon.0) 783 : audit [DBG] from='client.? 172.21.15.27:0/1141443849' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:05:55.959 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:54.589153+0000 mgr.y (mgr.14886) 123 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:55.959 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.493001+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.27:0/2933384842' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.959 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.493303+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.959 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:55.502978+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:54.589153+0000 mgr.y (mgr.14886) 123 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.493001+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.27:0/2933384842' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.493303+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:55.502978+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:55.671+0000 7fcd6d027640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T09:05:55.960 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ignoring --setuser ceph since I am not root 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ignoring --setgroup ceph since I am not root 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-mgr[52513]: -- 172.21.15.27:0/3791638583 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c68f500d00 con 0x55c68f4d4800 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:55.824+0000 7fcac2d89040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:55.886+0000 7fcac2d89040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:54.589153+0000 mgr.y (mgr.14886) 123 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.493001+0000 mon.c (mon.1) 69 : audit [INF] from='client.? 172.21.15.27:0/2933384842' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.493303+0000 mon.a (mon.0) 784 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-09-17T09:05:55.961 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:55.502978+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-17T09:05:56.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.045+0000 7fcac2d89040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:05:56.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.117+0000 7fcac2d89040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:05:56.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 180' 2024-09-17T09:05:56.676 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.273+0000 7fcac2d89040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:05:56.676 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.443+0000 7fcac2d89040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.792+0000 7fcac2d89040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:56.856+0000 7fcac2d89040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.669837+0000 mon.a (mon.0) 786 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:55.669891+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e27: x(active, starting, since 0.175924s) 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.673801+0000 mon.b (mon.2) 107 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:05:56.971 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.673975+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.674107+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.674444+0000 mon.b (mon.2) 110 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.674671+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.674765+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.677910+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.678130+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.678372+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.678585+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.678837+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.679085+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:05:56.972 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.679334+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.669837+0000 mon.a (mon.0) 786 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:55.669891+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e27: x(active, starting, since 0.175924s) 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.673801+0000 mon.b (mon.2) 107 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.673975+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.674107+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.674444+0000 mon.b (mon.2) 110 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.674671+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.674765+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.677910+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.678130+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:05:56.973 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.678372+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.678585+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.678837+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.679085+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.679334+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.679490+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.679927+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:55.781974+0000 mon.a (mon.0) 788 : cluster [INF] Manager daemon x is now available 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.800976+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.834969+0000 mon.b (mon.2) 123 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:56.974 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.835598+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.853592+0000 mon.b (mon.2) 124 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:55.854163+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:56.383709+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:56.389208+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:56.444839+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:05:56.451682+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.679490+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.679927+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:55.781974+0000 mon.a (mon.0) 788 : cluster [INF] Manager daemon x is now available 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.800976+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:56.975 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.834969+0000 mon.b (mon.2) 123 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.835598+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.853592+0000 mon.b (mon.2) 124 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:55.854163+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:56.383709+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:56.389208+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:56.444839+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:56.976 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:05:56.451682+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:57.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.669837+0000 mon.a (mon.0) 786 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:55.669891+0000 mon.a (mon.0) 787 : cluster [DBG] mgrmap e27: x(active, starting, since 0.175924s) 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.673801+0000 mon.b (mon.2) 107 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.673975+0000 mon.b (mon.2) 108 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.674107+0000 mon.b (mon.2) 109 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.674444+0000 mon.b (mon.2) 110 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.674671+0000 mon.b (mon.2) 111 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.674765+0000 mon.b (mon.2) 112 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.677910+0000 mon.b (mon.2) 113 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.678130+0000 mon.b (mon.2) 114 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.678372+0000 mon.b (mon.2) 115 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:05:57.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.678585+0000 mon.b (mon.2) 116 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.678837+0000 mon.b (mon.2) 117 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.679085+0000 mon.b (mon.2) 118 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.679334+0000 mon.b (mon.2) 119 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.679490+0000 mon.b (mon.2) 120 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.679927+0000 mon.b (mon.2) 121 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:55.781974+0000 mon.a (mon.0) 788 : cluster [INF] Manager daemon x is now available 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.800976+0000 mon.b (mon.2) 122 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.834969+0000 mon.b (mon.2) 123 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.835598+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.853592+0000 mon.b (mon.2) 124 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:55.854163+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:56.383709+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:56.389208+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:56.444839+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:05:56.451682+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:05:57.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:57.052+0000 7fcac2d89040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:05:57.860 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:56.684860+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e28: x(active, since 1.19089s) 2024-09-17T09:05:57.860 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:56.692080+0000 mgr.x (mgr.34718) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:05:57.565+0000 7f6ed7e97640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:09:05:57] ENGINE Error in HTTPServer.serve 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: Traceback (most recent call last): 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._connections.run(self.expiration_interval) 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._run(expiration_interval) 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:05:57.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: s = self.context.wrap_socket( 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: return self.sslsocket_class._create( 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self.do_handshake() 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: self._sslobj.do_handshake() 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:05:57.862 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:05:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: 2024-09-17T09:05:57.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:56.684860+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e28: x(active, since 1.19089s) 2024-09-17T09:05:57.983 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:56.692080+0000 mgr.x (mgr.34718) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:57.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:56.684860+0000 mon.a (mon.0) 795 : cluster [DBG] mgrmap e28: x(active, since 1.19089s) 2024-09-17T09:05:57.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:56.692080+0000 mgr.x (mgr.34718) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:58.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:57.994+0000 7fcac2d89040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:05:58.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:58.068+0000 7fcac2d89040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:05:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:58.857+0000 7fcac2d89040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:05:58.978 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:58.977+0000 7fcac2d89040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:05:58.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:57.446139+0000 mgr.x (mgr.34718) 2 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTING 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:57.562932+0000 mgr.x (mgr.34718) 3 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:57.566050+0000 mgr.x (mgr.34718) 4 : cephadm [ERR] [17/Sep/2024:09:05:57] ENGINE Error in HTTPServer.serve 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: Traceback (most recent call last): 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._connections.run(self.expiration_interval) 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._run(expiration_interval) 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:05:58.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s = self.context.wrap_socket( 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: return self.sslsocket_class._create( 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self.do_handshake() 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._sslobj.do_handshake() 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:57.664162+0000 mgr.x (mgr.34718) 5 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:05:57.664410+0000 mgr.x (mgr.34718) 6 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTED 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:57.677352+0000 mgr.x (mgr.34718) 7 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:57.681991+0000 mon.a (mon.0) 796 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:05:58.980 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:57.682014+0000 mon.a (mon.0) 797 : cluster [INF] Cluster is now healthy 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:57.446139+0000 mgr.x (mgr.34718) 2 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTING 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:57.562932+0000 mgr.x (mgr.34718) 3 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:57.566050+0000 mgr.x (mgr.34718) 4 : cephadm [ERR] [17/Sep/2024:09:05:57] ENGINE Error in HTTPServer.serve 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Traceback (most recent call last): 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._connections.run(self.expiration_interval) 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._run(expiration_interval) 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:05:58.981 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s = self.context.wrap_socket( 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: return self.sslsocket_class._create( 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self.do_handshake() 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._sslobj.do_handshake() 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:57.664162+0000 mgr.x (mgr.34718) 5 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T09:05:58.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:05:57.664410+0000 mgr.x (mgr.34718) 6 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTED 2024-09-17T09:05:58.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:57.677352+0000 mgr.x (mgr.34718) 7 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:58.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:57.681991+0000 mon.a (mon.0) 796 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:05:58.983 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:57.682014+0000 mon.a (mon.0) 797 : cluster [INF] Cluster is now healthy 2024-09-17T09:05:59.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:57.446139+0000 mgr.x (mgr.34718) 2 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTING 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:57.562932+0000 mgr.x (mgr.34718) 3 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on https://172.21.15.105:7150 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:57.566050+0000 mgr.x (mgr.34718) 4 : cephadm [ERR] [17/Sep/2024:09:05:57] ENGINE Error in HTTPServer.serve 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Traceback (most recent call last): 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._connections.run(self.expiration_interval) 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._run(expiration_interval) 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:05:59.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s = self.context.wrap_socket( 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: return self.sslsocket_class._create( 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self.do_handshake() 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._sslobj.do_handshake() 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:57.664162+0000 mgr.x (mgr.34718) 5 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Serving on http://172.21.15.105:8765 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:05:57.664410+0000 mgr.x (mgr.34718) 6 : cephadm [INF] [17/Sep/2024:09:05:57] ENGINE Bus STARTED 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:57.677352+0000 mgr.x (mgr.34718) 7 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:57.681991+0000 mon.a (mon.0) 796 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:05:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:57.682014+0000 mon.a (mon.0) 797 : cluster [INF] Cluster is now healthy 2024-09-17T09:05:59.257 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.127+0000 7fcac2d89040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:05:59.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.190+0000 7fcac2d89040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:05:59.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.352+0000 7fcac2d89040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:05:59.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.423+0000 7fcac2d89040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:05:59.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.485+0000 7fcac2d89040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:05:59.651 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.650+0000 7fcac2d89040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:05:59.939 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:58.821959+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e29: x(active, since 3s) 2024-09-17T09:05:59.940 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:58.821959+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e29: x(active, since 3s) 2024-09-17T09:05:59.940 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.720+0000 7fcac2d89040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:05:59.940 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.789+0000 7fcac2d89040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:05:59.940 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.877+0000 7fcac2d89040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:05:59.940 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:05:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:05:59.939+0000 7fcac2d89040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:06:00.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:05:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:58.821959+0000 mon.a (mon.0) 798 : cluster [DBG] mgrmap e29: x(active, since 3s) 2024-09-17T09:06:00.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:06:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:06:00.000+0000 7fcac2d89040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:06:01.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:06:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:06:00.928+0000 7fcac2d89040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:06:01.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:05:59.677849+0000 mgr.x (mgr.34718) 8 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:05:59.677849+0000 mgr.x (mgr.34718) 8 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:01.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:05:59.677849+0000 mgr.x (mgr.34718) 8 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:00.931042+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:00.932174+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:00.932700+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:00.933682+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:00.933979+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:01.220601+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:01.225683+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:01.722429+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:01.731335+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:00.931042+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:00.932174+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:00.932700+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:00.933682+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:00.933979+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:01.220601+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:01.225683+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:01.722429+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:01.731335+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:00.931042+0000 mon.a (mon.0) 799 : cluster [DBG] Standby manager daemon y started 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:00.932174+0000 mon.c (mon.1) 70 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:00.932700+0000 mon.c (mon.1) 71 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:00.933682+0000 mon.c (mon.1) 72 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:00.933979+0000 mon.c (mon.1) 73 : audit [DBG] from='mgr.? 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:01.220601+0000 mon.a (mon.0) 800 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:01.225683+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:01.722429+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:02.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:01.731335+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:03.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:01.678197+0000 mgr.x (mgr.34718) 9 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:01.839083+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e30: x(active, since 6s), standbys: y 2024-09-17T09:06:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:01.841142+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:06:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:01.678197+0000 mgr.x (mgr.34718) 9 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:01.839083+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e30: x(active, since 6s), standbys: y 2024-09-17T09:06:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:01.841142+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:06:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:01.678197+0000 mgr.x (mgr.34718) 9 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:01.839083+0000 mon.a (mon.0) 804 : cluster [DBG] mgrmap e30: x(active, since 6s), standbys: y 2024-09-17T09:06:03.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:01.841142+0000 mon.b (mon.2) 125 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:02.903233+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:02.910754+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:03.428749+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:03.434338+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:03.435991+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:03.436719+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:03.437467+0000 mgr.x (mgr.34718) 10 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:03.438435+0000 mgr.x (mgr.34718) 11 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:03.441551+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:03.518943+0000 mgr.x (mgr.34718) 12 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:06:04.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:03.744529+0000 mgr.x (mgr.34718) 13 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:02.903233+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:02.910754+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:03.428749+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:03.434338+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:03.435991+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:03.436719+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:03.437467+0000 mgr.x (mgr.34718) 10 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:03.438435+0000 mgr.x (mgr.34718) 11 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T09:06:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:03.441551+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:03.518943+0000 mgr.x (mgr.34718) 12 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:03.744529+0000 mgr.x (mgr.34718) 13 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:02.903233+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:02.910754+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:03.428749+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:03.434338+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:03.435991+0000 mon.b (mon.2) 126 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:03.436719+0000 mon.b (mon.2) 127 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:03.437467+0000 mgr.x (mgr.34718) 10 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:04.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:03.438435+0000 mgr.x (mgr.34718) 11 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-17T09:06:04.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:03.441551+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:04.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:03.518943+0000 mgr.x (mgr.34718) 12 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:06:04.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:03.744529+0000 mgr.x (mgr.34718) 13 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:06:05.180 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:04.433002+0000 mon.a (mon.0) 810 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:06:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:04.433002+0000 mon.a (mon.0) 810 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:06:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:04.433002+0000 mon.a (mon.0) 810 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:06:06.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:05.438891+0000 mgr.x (mgr.34718) 14 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:06:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:05.438891+0000 mgr.x (mgr.34718) 14 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:06:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:05.438891+0000 mgr.x (mgr.34718) 14 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:06:06.746 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:06 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:06:07.022 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:06 smithi105 podman[54718]: 2024-09-17 09:06:06.649402197 +0000 UTC m=+0.017070106 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:06:07.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:06 smithi105 podman[54718]: 2024-09-17 09:06:06.779764839 +0000 UTC m=+0.147432739 container create 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:07.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:06 smithi105 podman[54718]: 2024-09-17 09:06:06.990026159 +0000 UTC m=+0.357694059 container init 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:07.023 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:06 smithi105 podman[54718]: 2024-09-17 09:06:06.992736427 +0000 UTC m=+0.360404326 container start 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:07.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:06:07.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:06:07.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:06:07.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:06:07.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.022Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.023Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.023Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.025Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.026Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.035Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.035Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.035Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.035Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.036Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=12 2024-09-17T09:06:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.036Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=12 2024-09-17T09:06:07.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.036Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=12 2024-09-17T09:06:07.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.036Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=12 2024-09-17T09:06:07.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.037Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=12 2024-09-17T09:06:07.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.038Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=12 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.039Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=12 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.039Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=12 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.039Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=13.548564ms 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.041Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.041Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.041Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:06:07.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=info ts=2024-09-17T09:06:07.042Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[54733]: level=error ts=2024-09-17T09:06:07.042Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 bash[54718]: 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:07.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 podman[54751]: 2024-09-17 09:06:07.101195108 +0000 UTC m=+0.050860125 container died 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:07.945 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 podman[54751]: 2024-09-17 09:06:07.685868541 +0000 UTC m=+0.635533548 container remove 333f927bc9e93604821432405e9980ce802b16f3e6a519b6b347bcf547ec4e76 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:07.946 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:06:07.946 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:07 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:06:08.224 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:07.110818+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.224 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:07.117237+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:07.121936+0000 mgr.x (mgr.34718) 15 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:07.122228+0000 mon.b (mon.2) 128 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:07.122807+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:07.126324+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:07.127163+0000 mgr.x (mgr.34718) 16 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:06:08.225 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:07.439541+0000 mgr.x (mgr.34718) 17 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:07.110818+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:07.117237+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:07.121936+0000 mgr.x (mgr.34718) 15 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:07.122228+0000 mon.b (mon.2) 128 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:07.122807+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:07.126324+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:07.127163+0000 mgr.x (mgr.34718) 16 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:07.439541+0000 mgr.x (mgr.34718) 17 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:07.110818+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:07.117237+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:07.121936+0000 mgr.x (mgr.34718) 15 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:07.122228+0000 mon.b (mon.2) 128 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:07.122807+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:06:08.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:07.126324+0000 mon.b (mon.2) 129 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:08.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:07.127163+0000 mgr.x (mgr.34718) 16 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:06:08.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:07.439541+0000 mgr.x (mgr.34718) 17 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:11.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:09.439995+0000 mgr.x (mgr.34718) 18 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:11.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.693837+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.702760+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.707039+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.707409+0000 mgr.x (mgr.34718) 19 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.710355+0000 mon.b (mon.2) 131 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.710775+0000 mgr.x (mgr.34718) 20 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.713677+0000 mon.b (mon.2) 132 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.714014+0000 mgr.x (mgr.34718) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:09.753999+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:10.175966+0000 mon.b (mon.2) 134 : audit [DBG] from='client.? 172.21.15.105:0/1170566592' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:10.507201+0000 mon.b (mon.2) 135 : audit [INF] from='client.? 172.21.15.105:0/646926275' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:11.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:09.439995+0000 mgr.x (mgr.34718) 18 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.693837+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.702760+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.707039+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.707409+0000 mgr.x (mgr.34718) 19 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.710355+0000 mon.b (mon.2) 131 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.710775+0000 mgr.x (mgr.34718) 20 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.713677+0000 mon.b (mon.2) 132 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.714014+0000 mgr.x (mgr.34718) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:09.753999+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:10.175966+0000 mon.b (mon.2) 134 : audit [DBG] from='client.? 172.21.15.105:0/1170566592' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:10.507201+0000 mon.b (mon.2) 135 : audit [INF] from='client.? 172.21.15.105:0/646926275' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:09.439995+0000 mgr.x (mgr.34718) 18 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.693837+0000 mon.a (mon.0) 814 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.702760+0000 mon.a (mon.0) 815 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.707039+0000 mon.b (mon.2) 130 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.707409+0000 mgr.x (mgr.34718) 19 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.710355+0000 mon.b (mon.2) 131 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.710775+0000 mgr.x (mgr.34718) 20 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.713677+0000 mon.b (mon.2) 132 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.714014+0000 mgr.x (mgr.34718) 21 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:09.753999+0000 mon.b (mon.2) 133 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:10.175966+0000 mon.b (mon.2) 134 : audit [DBG] from='client.? 172.21.15.105:0/1170566592' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:10.507201+0000 mon.b (mon.2) 135 : audit [INF] from='client.? 172.21.15.105:0/646926275' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:11.440321+0000 mgr.x (mgr.34718) 22 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:13.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:11.440321+0000 mgr.x (mgr.34718) 22 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:13.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:11.440321+0000 mgr.x (mgr.34718) 22 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:06:15.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:13.441026+0000 mgr.x (mgr.34718) 23 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:13.441026+0000 mgr.x (mgr.34718) 23 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:13.441026+0000 mgr.x (mgr.34718) 23 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:15.441405+0000 mgr.x (mgr.34718) 24 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:15.621697+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:15.627826+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:15.628862+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:15.629440+0000 mon.b (mon.2) 137 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:06:15.629951+0000 mgr.x (mgr.34718) 25 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:15.630385+0000 mgr.x (mgr.34718) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 100 B/s rd, 0 op/s 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:15.630563+0000 mgr.x (mgr.34718) 27 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:15.635070+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:15.441405+0000 mgr.x (mgr.34718) 24 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:15.621697+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:15.627826+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:15.628862+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:15.629440+0000 mon.b (mon.2) 137 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:17.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:06:15.629951+0000 mgr.x (mgr.34718) 25 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:15.630385+0000 mgr.x (mgr.34718) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 100 B/s rd, 0 op/s 2024-09-17T09:06:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:15.630563+0000 mgr.x (mgr.34718) 27 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:17.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:15.635070+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:15.441405+0000 mgr.x (mgr.34718) 24 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:17.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:15.621697+0000 mon.a (mon.0) 816 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:15.627826+0000 mon.a (mon.0) 817 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:15.628862+0000 mon.b (mon.2) 136 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:15.629440+0000 mon.b (mon.2) 137 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:06:15.629951+0000 mgr.x (mgr.34718) 25 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:15.630385+0000 mgr.x (mgr.34718) 26 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 100 B/s rd, 0 op/s 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:15.630563+0000 mgr.x (mgr.34718) 27 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:15.635070+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:06:18.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:17 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 1. 2024-09-17T09:06:18.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:17 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:18.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:17 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:06:18.028 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:17 smithi105 podman[55860]: 2024-09-17 09:06:17.800196009 +0000 UTC m=+0.017057044 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:06:18.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:17 smithi105 podman[55860]: 2024-09-17 09:06:17.929382883 +0000 UTC m=+0.146243919 container create 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:18.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 podman[55860]: 2024-09-17 09:06:18.156042898 +0000 UTC m=+0.372903934 container init 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:18.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 podman[55860]: 2024-09-17 09:06:18.15863122 +0000 UTC m=+0.375492258 container start 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.188Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.188Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.188Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.188Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.188Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.189Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.190Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.190Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.193Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.193Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=13 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.201Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=13 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.201Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=13 2024-09-17T09:06:18.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.201Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.202Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.202Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.203Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.203Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.203Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=13 2024-09-17T09:06:18.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=13 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=13 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=13 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=13 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.204Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=11.127512ms 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:06:18.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=info ts=2024-09-17T09:06:18.206Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[55875]: level=error ts=2024-09-17T09:06:18.207Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 bash[55860]: 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:18.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 podman[55893]: 2024-09-17 09:06:18.23847904 +0000 UTC m=+0.024352494 container died 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:17.631376+0000 mgr.x (mgr.34718) 28 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:17.631376+0000 mgr.x (mgr.34718) 28 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:17.631376+0000 mgr.x (mgr.34718) 28 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:19.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 podman[55893]: 2024-09-17 09:06:18.793368585 +0000 UTC m=+0.579242077 container remove 368dfa338c4ea6b2ac5d3302e962a0071aa710fd95d1297356a7c86409f21636 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:19.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:06:19.279 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:18 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:06:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:19.631878+0000 mgr.x (mgr.34718) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:19.631878+0000 mgr.x (mgr.34718) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:19.631878+0000 mgr.x (mgr.34718) 29 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 125 B/s rd, 0 op/s 2024-09-17T09:06:22.934 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:21.632305+0000 mgr.x (mgr.34718) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:21.632305+0000 mgr.x (mgr.34718) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:21.632305+0000 mgr.x (mgr.34718) 30 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:23.620415+0000 mon.b (mon.2) 138 : audit [DBG] from='client.? 172.21.15.105:0/932875948' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:24.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:23.620415+0000 mon.b (mon.2) 138 : audit [DBG] from='client.? 172.21.15.105:0/932875948' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:23.620415+0000 mon.b (mon.2) 138 : audit [DBG] from='client.? 172.21.15.105:0/932875948' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:25.015 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:23.632982+0000 mgr.x (mgr.34718) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:25.015 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:23.935922+0000 mon.b (mon.2) 139 : audit [INF] from='client.? 172.21.15.105:0/4031049857' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:23.632982+0000 mgr.x (mgr.34718) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:23.935922+0000 mon.b (mon.2) 139 : audit [INF] from='client.? 172.21.15.105:0/4031049857' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:23.632982+0000 mgr.x (mgr.34718) 31 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:23.935922+0000 mon.b (mon.2) 139 : audit [INF] from='client.? 172.21.15.105:0/4031049857' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:27.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:25.633364+0000 mgr.x (mgr.34718) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:25.633364+0000 mgr.x (mgr.34718) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:25.633364+0000 mgr.x (mgr.34718) 32 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:27.634243+0000 mgr.x (mgr.34718) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:27.634243+0000 mgr.x (mgr.34718) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:27.634243+0000 mgr.x (mgr.34718) 33 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:29.347 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 2. 2024-09-17T09:06:29.347 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:29.347 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:06:29.347 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 podman[56159]: 2024-09-17 09:06:29.115359915 +0000 UTC m=+0.016874418 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:06:29.347 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 podman[56159]: 2024-09-17 09:06:29.232985254 +0000 UTC m=+0.134499747 container create 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:29.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 podman[56159]: 2024-09-17 09:06:29.459940878 +0000 UTC m=+0.361455373 container init 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 podman[56159]: 2024-09-17 09:06:29.463097487 +0000 UTC m=+0.364611976 container start 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.500Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.501Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.501Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.504Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.505Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=14 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=14 2024-09-17T09:06:29.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.514Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=14 2024-09-17T09:06:29.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.515Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=14 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.516Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=11.158719ms 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.517Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.517Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.517Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:06:29.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=info ts=2024-09-17T09:06:29.518Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56174]: level=error ts=2024-09-17T09:06:29.518Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 bash[56159]: 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:29.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:29 smithi105 podman[56193]: 2024-09-17 09:06:29.560064354 +0000 UTC m=+0.034494775 container died 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:30.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:30 smithi105 podman[56193]: 2024-09-17 09:06:30.155404321 +0000 UTC m=+0.629834741 container remove 7799aebaafebe1b7af934af65449e2ae60ded8512ae0a649f2a4dc0f7a6e77ed (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:30.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:30 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:06:30.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:30 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:06:31.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:29.634633+0000 mgr.x (mgr.34718) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:29.634633+0000 mgr.x (mgr.34718) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:29.634633+0000 mgr.x (mgr.34718) 34 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:31.635155+0000 mgr.x (mgr.34718) 35 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:33.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:31.635155+0000 mgr.x (mgr.34718) 35 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:31.635155+0000 mgr.x (mgr.34718) 35 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:33.636173+0000 mgr.x (mgr.34718) 36 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:33.636173+0000 mgr.x (mgr.34718) 36 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:33.636173+0000 mgr.x (mgr.34718) 36 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:35.636551+0000 mgr.x (mgr.34718) 37 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:36.714624+0000 mon.b (mon.2) 140 : audit [DBG] from='client.? 172.21.15.105:0/1023742706' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:37.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:35.636551+0000 mgr.x (mgr.34718) 37 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:37.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:36.714624+0000 mon.b (mon.2) 140 : audit [DBG] from='client.? 172.21.15.105:0/1023742706' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:35.636551+0000 mgr.x (mgr.34718) 37 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:36.714624+0000 mon.b (mon.2) 140 : audit [DBG] from='client.? 172.21.15.105:0/1023742706' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:38.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:37.028715+0000 mon.b (mon.2) 141 : audit [INF] from='client.? 172.21.15.105:0/1860311639' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:37.028715+0000 mon.b (mon.2) 141 : audit [INF] from='client.? 172.21.15.105:0/1860311639' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:37.028715+0000 mon.b (mon.2) 141 : audit [INF] from='client.? 172.21.15.105:0/1860311639' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:37.637431+0000 mgr.x (mgr.34718) 38 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:06:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:37.637431+0000 mgr.x (mgr.34718) 38 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:06:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:37.637431+0000 mgr.x (mgr.34718) 38 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:06:40.569 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 3. 2024-09-17T09:06:40.569 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:40.569 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:06:40.569 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 podman[56458]: 2024-09-17 09:06:40.366762534 +0000 UTC m=+0.017448222 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:06:40.569 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 podman[56458]: 2024-09-17 09:06:40.470171177 +0000 UTC m=+0.120856853 container create 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:40.877 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 podman[56458]: 2024-09-17 09:06:40.663442238 +0000 UTC m=+0.314127916 container init 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 podman[56458]: 2024-09-17 09:06:40.666493168 +0000 UTC m=+0.317178848 container start 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.697Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.698Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.698Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.701Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.701Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=15 2024-09-17T09:06:40.878 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.709Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.709Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.709Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.709Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.710Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.710Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=15 2024-09-17T09:06:40.879 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.710Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.710Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.710Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.711Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.711Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.713Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.713Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=15 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.713Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=11.979065ms 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.714Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.714Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.714Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:06:40.880 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=info ts=2024-09-17T09:06:40.715Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56472]: level=error ts=2024-09-17T09:06:40.715Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 bash[56458]: 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:40.881 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:40 smithi105 podman[56489]: 2024-09-17 09:06:40.747544093 +0000 UTC m=+0.024372014 container died 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:41.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:39.637958+0000 mgr.x (mgr.34718) 39 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:41.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:39.637958+0000 mgr.x (mgr.34718) 39 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:41.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:39.637958+0000 mgr.x (mgr.34718) 39 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:41.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:41 smithi105 podman[56489]: 2024-09-17 09:06:41.284145387 +0000 UTC m=+0.560973289 container remove 3b4688b2d07379b5193e603c4a3f8fc82661f257ec413abfd39b551e2524642c (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:41.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:41 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:06:41.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:41 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:06:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:41.638617+0000 mgr.x (mgr.34718) 40 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:41.638617+0000 mgr.x (mgr.34718) 40 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:41.638617+0000 mgr.x (mgr.34718) 40 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:43.639392+0000 mgr.x (mgr.34718) 41 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:43.639392+0000 mgr.x (mgr.34718) 41 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:43.639392+0000 mgr.x (mgr.34718) 41 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:45.639858+0000 mgr.x (mgr.34718) 42 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:45.639858+0000 mgr.x (mgr.34718) 42 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:45.639858+0000 mgr.x (mgr.34718) 42 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:48.922 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:47.640755+0000 mgr.x (mgr.34718) 43 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:49.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:47.640755+0000 mgr.x (mgr.34718) 43 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:47.640755+0000 mgr.x (mgr.34718) 43 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:50.109 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:49.750006+0000 mon.b (mon.2) 142 : audit [DBG] from='client.? 172.21.15.105:0/2516250246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:49.750006+0000 mon.b (mon.2) 142 : audit [DBG] from='client.? 172.21.15.105:0/2516250246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:49.750006+0000 mon.b (mon.2) 142 : audit [DBG] from='client.? 172.21.15.105:0/2516250246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:06:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:49.641161+0000 mgr.x (mgr.34718) 44 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:06:50.067058+0000 mon.b (mon.2) 143 : audit [INF] from='client.? 172.21.15.105:0/3296952485' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:51.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:49.641161+0000 mgr.x (mgr.34718) 44 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:51.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:06:50.067058+0000 mon.b (mon.2) 143 : audit [INF] from='client.? 172.21.15.105:0/3296952485' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:51.160 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:49.641161+0000 mgr.x (mgr.34718) 44 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:51.160 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:06:50.067058+0000 mon.b (mon.2) 143 : audit [INF] from='client.? 172.21.15.105:0/3296952485' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:06:51.659 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 4. 2024-09-17T09:06:51.659 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:51.659 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:06:51.659 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 podman[56760]: 2024-09-17 09:06:51.405260952 +0000 UTC m=+0.017946875 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:06:51.659 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 podman[56760]: 2024-09-17 09:06:51.517095364 +0000 UTC m=+0.129781276 container create cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:52.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 podman[56760]: 2024-09-17 09:06:51.777385655 +0000 UTC m=+0.390071575 container init cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:52.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 podman[56760]: 2024-09-17 09:06:51.780152949 +0000 UTC m=+0.392838861 container start cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:52.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:06:52.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:06:52.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.810Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.811Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.811Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.813Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:06:52.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.814Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.821Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=16 2024-09-17T09:06:52.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.822Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.823Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.823Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.824Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.825Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.825Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=16 2024-09-17T09:06:52.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.825Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=16 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.825Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=16 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.825Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=11.959365ms 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:06:52.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.827Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.828Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:06:52.034 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=info ts=2024-09-17T09:06:51.828Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:06:52.035 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[56776]: level=error ts=2024-09-17T09:06:51.828Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:06:52.035 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 bash[56760]: cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 2024-09-17T09:06:52.035 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:06:52.035 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:51 smithi105 podman[56794]: 2024-09-17 09:06:51.860564978 +0000 UTC m=+0.025249192 container died cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:52.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:52 smithi105 podman[56794]: 2024-09-17 09:06:52.397513319 +0000 UTC m=+0.562197536 container remove cbdeb1942369792816a91e94b0ecb6959a9d7490e867011ffb98901b8d2f1af3 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:06:52.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:52 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:06:52.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:06:52 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:06:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:51.641526+0000 mgr.x (mgr.34718) 45 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:51.641526+0000 mgr.x (mgr.34718) 45 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:51.641526+0000 mgr.x (mgr.34718) 45 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:06:55.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:53.642472+0000 mgr.x (mgr.34718) 46 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:53.642472+0000 mgr.x (mgr.34718) 46 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:53.642472+0000 mgr.x (mgr.34718) 46 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:57.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:55.642880+0000 mgr.x (mgr.34718) 47 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:55.642880+0000 mgr.x (mgr.34718) 47 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:55.642880+0000 mgr.x (mgr.34718) 47 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:06:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:57.643714+0000 mgr.x (mgr.34718) 48 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:06:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:57.643714+0000 mgr.x (mgr.34718) 48 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:06:59.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:06:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:57.643714+0000 mgr.x (mgr.34718) 48 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:06:59.644276+0000 mgr.x (mgr.34718) 49 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:01.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:06:59.644276+0000 mgr.x (mgr.34718) 49 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:01.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:06:59.644276+0000 mgr.x (mgr.34718) 49 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:02.862 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:07:02 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 5. 2024-09-17T09:07:02.862 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:07:02 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:07:02.862 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:07:02 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Start request repeated too quickly. 2024-09-17T09:07:02.862 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:07:02 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:07:02.863 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:07:02 smithi105 systemd[1]: Failed to start Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:07:03.240 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:01.644625+0000 mgr.x (mgr.34718) 50 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:03.240 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:02.620060+0000 mon.b (mon.2) 144 : audit [DBG] from='client.? 172.21.15.105:0/1237341393' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:07:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:01.644625+0000 mgr.x (mgr.34718) 50 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:03.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:02.620060+0000 mon.b (mon.2) 144 : audit [DBG] from='client.? 172.21.15.105:0/1237341393' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:07:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:01.644625+0000 mgr.x (mgr.34718) 50 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:03.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:02.620060+0000 mon.b (mon.2) 144 : audit [DBG] from='client.? 172.21.15.105:0/1237341393' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:07:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:02.936263+0000 mon.b (mon.2) 145 : audit [INF] from='client.? 172.21.15.105:0/1660997543' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:07:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:02.936263+0000 mon.b (mon.2) 145 : audit [INF] from='client.? 172.21.15.105:0/1660997543' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:07:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:02.936263+0000 mon.b (mon.2) 145 : audit [INF] from='client.? 172.21.15.105:0/1660997543' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:07:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:03.645476+0000 mgr.x (mgr.34718) 51 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:03.645476+0000 mgr.x (mgr.34718) 51 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:03.645476+0000 mgr.x (mgr.34718) 51 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:05.645964+0000 mgr.x (mgr.34718) 52 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:05.645964+0000 mgr.x (mgr.34718) 52 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:05.645964+0000 mgr.x (mgr.34718) 52 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:07.647000+0000 mgr.x (mgr.34718) 53 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:07.647000+0000 mgr.x (mgr.34718) 53 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:07.647000+0000 mgr.x (mgr.34718) 53 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:09.647407+0000 mgr.x (mgr.34718) 54 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:09.647407+0000 mgr.x (mgr.34718) 54 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:09.647407+0000 mgr.x (mgr.34718) 54 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:13.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:11.647911+0000 mgr.x (mgr.34718) 55 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:11.647911+0000 mgr.x (mgr.34718) 55 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:11.647911+0000 mgr.x (mgr.34718) 55 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:13.648777+0000 mgr.x (mgr.34718) 56 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:13.648777+0000 mgr.x (mgr.34718) 56 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:13.648777+0000 mgr.x (mgr.34718) 56 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:15.649268+0000 mgr.x (mgr.34718) 57 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:15.748132+0000 mon.b (mon.2) 146 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:07:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:15.649268+0000 mgr.x (mgr.34718) 57 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:15.748132+0000 mon.b (mon.2) 146 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:07:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:15.649268+0000 mgr.x (mgr.34718) 57 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:15.748132+0000 mon.b (mon.2) 146 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:07:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:17.649916+0000 mgr.x (mgr.34718) 58 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:19.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:17.649916+0000 mgr.x (mgr.34718) 58 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:17.649916+0000 mgr.x (mgr.34718) 58 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:07:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:19.650308+0000 mgr.x (mgr.34718) 59 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:20.470601+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:20.476444+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:20.769032+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:07:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:20.769715+0000 mon.b (mon.2) 148 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:07:20.774858+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:19.650308+0000 mgr.x (mgr.34718) 59 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:20.470601+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:20.476444+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:20.769032+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:20.769715+0000 mon.b (mon.2) 148 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:07:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:07:20.774858+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:19.650308+0000 mgr.x (mgr.34718) 59 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:20.470601+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:20.476444+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:20.769032+0000 mon.b (mon.2) 147 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:20.769715+0000 mon.b (mon.2) 148 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:07:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:07:20.774858+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:07:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:07:20.770271+0000 mgr.x (mgr.34718) 60 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:07:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:20.770719+0000 mgr.x (mgr.34718) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:07:20.770271+0000 mgr.x (mgr.34718) 60 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:07:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:20.770719+0000 mgr.x (mgr.34718) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:07:20.770271+0000 mgr.x (mgr.34718) 60 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:07:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:20.770719+0000 mgr.x (mgr.34718) 61 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:24.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:22.771534+0000 mgr.x (mgr.34718) 62 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:24.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:22.771534+0000 mgr.x (mgr.34718) 62 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:22.771534+0000 mgr.x (mgr.34718) 62 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:24.772131+0000 mgr.x (mgr.34718) 63 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:24.772131+0000 mgr.x (mgr.34718) 63 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:24.772131+0000 mgr.x (mgr.34718) 63 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:26.772906+0000 mgr.x (mgr.34718) 64 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:26.772906+0000 mgr.x (mgr.34718) 64 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:26.772906+0000 mgr.x (mgr.34718) 64 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:30.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:28.773369+0000 mgr.x (mgr.34718) 65 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:30.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:28.773369+0000 mgr.x (mgr.34718) 65 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:30.260 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:28.773369+0000 mgr.x (mgr.34718) 65 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:32.257 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:30.773782+0000 mgr.x (mgr.34718) 66 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:30.773782+0000 mgr.x (mgr.34718) 66 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:32.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:30.773782+0000 mgr.x (mgr.34718) 66 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:34.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:32.774525+0000 mgr.x (mgr.34718) 67 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:32.774525+0000 mgr.x (mgr.34718) 67 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:32.774525+0000 mgr.x (mgr.34718) 67 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:34.775172+0000 mgr.x (mgr.34718) 68 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:34.775172+0000 mgr.x (mgr.34718) 68 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:34.775172+0000 mgr.x (mgr.34718) 68 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:36.776003+0000 mgr.x (mgr.34718) 69 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:36.776003+0000 mgr.x (mgr.34718) 69 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:36.776003+0000 mgr.x (mgr.34718) 69 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:40.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:38.776426+0000 mgr.x (mgr.34718) 70 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:40.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:38.776426+0000 mgr.x (mgr.34718) 70 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:38.776426+0000 mgr.x (mgr.34718) 70 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:40.776937+0000 mgr.x (mgr.34718) 71 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:40.776937+0000 mgr.x (mgr.34718) 71 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:40.776937+0000 mgr.x (mgr.34718) 71 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:42.777715+0000 mgr.x (mgr.34718) 72 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:42.777715+0000 mgr.x (mgr.34718) 72 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:42.777715+0000 mgr.x (mgr.34718) 72 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:44.778220+0000 mgr.x (mgr.34718) 73 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:44.778220+0000 mgr.x (mgr.34718) 73 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:44.778220+0000 mgr.x (mgr.34718) 73 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:46.779028+0000 mgr.x (mgr.34718) 74 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:46.779028+0000 mgr.x (mgr.34718) 74 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:46.779028+0000 mgr.x (mgr.34718) 74 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:48.779603+0000 mgr.x (mgr.34718) 75 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:48.779603+0000 mgr.x (mgr.34718) 75 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:48.779603+0000 mgr.x (mgr.34718) 75 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:51.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:50.780245+0000 mgr.x (mgr.34718) 76 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:50.780245+0000 mgr.x (mgr.34718) 76 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:50.780245+0000 mgr.x (mgr.34718) 76 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:52.781051+0000 mgr.x (mgr.34718) 77 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:52.781051+0000 mgr.x (mgr.34718) 77 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:52.781051+0000 mgr.x (mgr.34718) 77 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:56.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:54.781636+0000 mgr.x (mgr.34718) 78 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:56.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:54.781636+0000 mgr.x (mgr.34718) 78 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:54.781636+0000 mgr.x (mgr.34718) 78 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:56.782584+0000 mgr.x (mgr.34718) 79 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:56.782584+0000 mgr.x (mgr.34718) 79 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:07:58.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:56.782584+0000 mgr.x (mgr.34718) 79 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:00.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:07:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:07:58.783012+0000 mgr.x (mgr.34718) 80 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:00.956 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:07:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:07:58.783012+0000 mgr.x (mgr.34718) 80 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:00.956 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:07:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:07:58.783012+0000 mgr.x (mgr.34718) 80 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:02.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:00.783557+0000 mgr.x (mgr.34718) 81 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:02.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:00.783557+0000 mgr.x (mgr.34718) 81 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:02.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:00.783557+0000 mgr.x (mgr.34718) 81 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:04.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:02.784480+0000 mgr.x (mgr.34718) 82 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:02.784480+0000 mgr.x (mgr.34718) 82 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:04.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:02.784480+0000 mgr.x (mgr.34718) 82 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:04.785201+0000 mgr.x (mgr.34718) 83 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:04.785201+0000 mgr.x (mgr.34718) 83 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:04.785201+0000 mgr.x (mgr.34718) 83 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:06.785921+0000 mgr.x (mgr.34718) 84 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:08.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:06.785921+0000 mgr.x (mgr.34718) 84 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:06.785921+0000 mgr.x (mgr.34718) 84 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:08.786281+0000 mgr.x (mgr.34718) 85 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:08.786281+0000 mgr.x (mgr.34718) 85 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:10.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:08.786281+0000 mgr.x (mgr.34718) 85 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:10.786877+0000 mgr.x (mgr.34718) 86 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:10.786877+0000 mgr.x (mgr.34718) 86 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:10.786877+0000 mgr.x (mgr.34718) 86 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:14.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:12.787618+0000 mgr.x (mgr.34718) 87 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:14.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:12.787618+0000 mgr.x (mgr.34718) 87 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:14.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:12.787618+0000 mgr.x (mgr.34718) 87 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:14.788268+0000 mgr.x (mgr.34718) 88 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:14.788268+0000 mgr.x (mgr.34718) 88 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:16.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:14.788268+0000 mgr.x (mgr.34718) 88 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:16.789204+0000 mgr.x (mgr.34718) 89 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:16.789204+0000 mgr.x (mgr.34718) 89 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:16.789204+0000 mgr.x (mgr.34718) 89 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:19.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:18.789765+0000 mgr.x (mgr.34718) 90 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:18.789765+0000 mgr.x (mgr.34718) 90 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:18.789765+0000 mgr.x (mgr.34718) 90 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:20.790409+0000 mgr.x (mgr.34718) 91 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:08:20.888116+0000 mon.b (mon.2) 149 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:08:21.235540+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:08:21.236193+0000 mon.b (mon.2) 151 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:08:21.241897+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:08:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:20.790409+0000 mgr.x (mgr.34718) 91 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:22.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:08:20.888116+0000 mon.b (mon.2) 149 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:08:22.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:08:21.235540+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:08:22.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:08:21.236193+0000 mon.b (mon.2) 151 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:08:22.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:08:21.241897+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:08:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:20.790409+0000 mgr.x (mgr.34718) 91 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:08:20.888116+0000 mon.b (mon.2) 149 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:08:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:08:21.235540+0000 mon.b (mon.2) 150 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:08:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:08:21.236193+0000 mon.b (mon.2) 151 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:08:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:08:21.241897+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:08:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:08:21.236676+0000 mgr.x (mgr.34718) 92 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:08:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:21.237172+0000 mgr.x (mgr.34718) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:21.237386+0000 mgr.x (mgr.34718) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:08:21.236676+0000 mgr.x (mgr.34718) 92 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:08:23.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:21.237172+0000 mgr.x (mgr.34718) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:23.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:21.237386+0000 mgr.x (mgr.34718) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:08:21.236676+0000 mgr.x (mgr.34718) 92 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:08:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:21.237172+0000 mgr.x (mgr.34718) 93 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:21.237386+0000 mgr.x (mgr.34718) 94 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:25.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:23.238162+0000 mgr.x (mgr.34718) 95 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:23.238162+0000 mgr.x (mgr.34718) 95 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:25.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:23.238162+0000 mgr.x (mgr.34718) 95 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:25.238879+0000 mgr.x (mgr.34718) 96 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:27.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:25.238879+0000 mgr.x (mgr.34718) 96 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:25.238879+0000 mgr.x (mgr.34718) 96 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:27.239892+0000 mgr.x (mgr.34718) 97 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:27.239892+0000 mgr.x (mgr.34718) 97 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:27.239892+0000 mgr.x (mgr.34718) 97 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:29.240255+0000 mgr.x (mgr.34718) 98 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:31.898 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:29.240255+0000 mgr.x (mgr.34718) 98 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:31.899 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:29.240255+0000 mgr.x (mgr.34718) 98 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:33.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:31.240753+0000 mgr.x (mgr.34718) 99 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:33.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:31.240753+0000 mgr.x (mgr.34718) 99 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:31.240753+0000 mgr.x (mgr.34718) 99 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:33.241738+0000 mgr.x (mgr.34718) 100 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:33.241738+0000 mgr.x (mgr.34718) 100 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:33.241738+0000 mgr.x (mgr.34718) 100 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:35.242238+0000 mgr.x (mgr.34718) 101 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:37.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:35.242238+0000 mgr.x (mgr.34718) 101 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:37.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:35.242238+0000 mgr.x (mgr.34718) 101 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:37.243034+0000 mgr.x (mgr.34718) 102 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:37.243034+0000 mgr.x (mgr.34718) 102 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:37.243034+0000 mgr.x (mgr.34718) 102 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:41.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:39.243702+0000 mgr.x (mgr.34718) 103 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:41.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:39.243702+0000 mgr.x (mgr.34718) 103 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:41.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:39.243702+0000 mgr.x (mgr.34718) 103 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:43.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:41.244286+0000 mgr.x (mgr.34718) 104 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:41.244286+0000 mgr.x (mgr.34718) 104 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:41.244286+0000 mgr.x (mgr.34718) 104 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:43.245081+0000 mgr.x (mgr.34718) 105 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:43.245081+0000 mgr.x (mgr.34718) 105 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:45.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:43.245081+0000 mgr.x (mgr.34718) 105 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:45.245565+0000 mgr.x (mgr.34718) 106 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:45.245565+0000 mgr.x (mgr.34718) 106 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:47.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:45.245565+0000 mgr.x (mgr.34718) 106 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:49.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:47.246338+0000 mgr.x (mgr.34718) 107 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:47.246338+0000 mgr.x (mgr.34718) 107 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:49.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:47.246338+0000 mgr.x (mgr.34718) 107 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:51.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:49.246994+0000 mgr.x (mgr.34718) 108 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:49.246994+0000 mgr.x (mgr.34718) 108 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:51.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:49.246994+0000 mgr.x (mgr.34718) 108 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:53.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:51.247577+0000 mgr.x (mgr.34718) 109 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:51.247577+0000 mgr.x (mgr.34718) 109 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:51.247577+0000 mgr.x (mgr.34718) 109 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:53.248481+0000 mgr.x (mgr.34718) 110 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:53.248481+0000 mgr.x (mgr.34718) 110 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:54.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:53.248481+0000 mgr.x (mgr.34718) 110 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:56.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:55.249033+0000 mgr.x (mgr.34718) 111 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:56.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:55.249033+0000 mgr.x (mgr.34718) 111 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:56.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:55.249033+0000 mgr.x (mgr.34718) 111 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:58.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:08:59.187 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:08:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:57.249976+0000 mgr.x (mgr.34718) 112 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:59.187 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:08:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:57.249976+0000 mgr.x (mgr.34718) 112 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:08:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:08:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:57.249976+0000 mgr.x (mgr.34718) 112 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:00.815 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (12m) 2m ago 20m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (12m) 100s ago 19m 35.3M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 100s ago 19m - - 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443 running (15m) 100s ago 24m 477M - 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (9m) 2m ago 25m 214M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (25m) 2m ago 25m 79.5M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (24m) 100s ago 24m 68.3M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (24m) 2m ago 24m 54.5M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 2m ago 20m 15.0M - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (12m) 100s ago 20m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T09:09:00.816 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (23m) 2m ago 23m 48.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (23m) 2m ago 23m 49.3M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (23m) 2m ago 23m 44.3M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (22m) 2m ago 22m 48.5M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (22m) 100s ago 22m 51.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (22m) 100s ago 22m 45.9M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (21m) 100s ago 21m 48.0M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (21m) 100s ago 21m 52.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 error 100s ago 21m - - 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (19m) 2m ago 19m 40.2M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:09:00.817 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (19m) 100s ago 19m 40.9M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:09:01.117 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:08:59.250528+0000 mgr.x (mgr.34718) 113 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:01.117 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:08:59.250528+0000 mgr.x (mgr.34718) 113 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:01.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:08:59.250528+0000 mgr.x (mgr.34718) 113 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:01.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ls' 2024-09-17T09:09:02.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:00.806030+0000 mgr.x (mgr.34718) 114 : audit [DBG] from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:02.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:00.806030+0000 mgr.x (mgr.34718) 114 : audit [DBG] from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:00.806030+0000 mgr.x (mgr.34718) 114 : audit [DBG] from='client.25129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:03.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:01.251026+0000 mgr.x (mgr.34718) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:03.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:01.251026+0000 mgr.x (mgr.34718) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:03.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:01.251026+0000 mgr.x (mgr.34718) 115 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi027=a;count:1 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:grafana ?:3000 1/1 102s ago 21m smithi105=a;count:1 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo ?:5000 0/1 102s ago 19m count:1 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:mgr 2/2 3m ago 24m smithi027=y;smithi105=x;count:2 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:mon 3/3 3m ago 24m smithi027:172.21.15.27=a;smithi027:[v2:172.21.15.27:3301,v1:172.21.15.27:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-17T09:09:03.460 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi027=a;smithi105=b;count:2 2024-09-17T09:09:03.461 INFO:teuthology.orchestra.run.smithi027.stdout:osd 8 3m ago - 2024-09-17T09:09:03.461 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus ?:9095 0/1 102s ago 21m smithi105=a;count:1 2024-09-17T09:09:03.461 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-09-17T09:09:04.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T09:09:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:03.251939+0000 mgr.x (mgr.34718) 116 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:05.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:03.454596+0000 mgr.x (mgr.34718) 117 : audit [DBG] from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:03.251939+0000 mgr.x (mgr.34718) 116 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:03.454596+0000 mgr.x (mgr.34718) 117 : audit [DBG] from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:03.251939+0000 mgr.x (mgr.34718) 116 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:03.454596+0000 mgr.x (mgr.34718) 117 : audit [DBG] from='client.15195 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:09:06.094 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:09:06.095 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:09:06.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-17T09:09:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:05.252391+0000 mgr.x (mgr.34718) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:06.093256+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.27:0/817784416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:05.252391+0000 mgr.x (mgr.34718) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:06.093256+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.27:0/817784416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:05.252391+0000 mgr.x (mgr.34718) 118 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:06.093256+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.27:0/817784416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:09.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:07.253188+0000 mgr.x (mgr.34718) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:07.253188+0000 mgr.x (mgr.34718) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:07.253188+0000 mgr.x (mgr.34718) 119 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:10.236 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:10.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:08.983939+0000 mgr.x (mgr.34718) 120 : audit [DBG] from='client.25147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:09.253774+0000 mgr.x (mgr.34718) 121 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:10.234970+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:10.236004+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:10.237458+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:10.238109+0000 mon.b (mon.2) 154 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:11.100 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:10.246182+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:08.983939+0000 mgr.x (mgr.34718) 120 : audit [DBG] from='client.25147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:09.253774+0000 mgr.x (mgr.34718) 121 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:10.234970+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:10.236004+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:10.237458+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:10.238109+0000 mon.b (mon.2) 154 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:11.101 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:10.246182+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:08.983939+0000 mgr.x (mgr.34718) 120 : audit [DBG] from='client.25147 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:09.253774+0000 mgr.x (mgr.34718) 121 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:10.234970+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:10.236004+0000 mon.b (mon.2) 152 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:10.237458+0000 mon.b (mon.2) 153 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:10.238109+0000 mon.b (mon.2) 154 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:11.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:10.246182+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:12.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:10.228106+0000 mgr.x (mgr.34718) 122 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:12.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:10.238688+0000 mgr.x (mgr.34718) 123 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:12.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:10.239215+0000 mgr.x (mgr.34718) 124 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:10.239376+0000 mgr.x (mgr.34718) 125 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.153 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:10.360603+0000 mgr.x (mgr.34718) 126 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:12.154 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:10.228106+0000 mgr.x (mgr.34718) 122 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:12.154 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:10.238688+0000 mgr.x (mgr.34718) 123 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:12.154 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:10.239215+0000 mgr.x (mgr.34718) 124 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.154 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:10.239376+0000 mgr.x (mgr.34718) 125 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.154 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:10.360603+0000 mgr.x (mgr.34718) 126 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:10.228106+0000 mgr.x (mgr.34718) 122 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:10.238688+0000 mgr.x (mgr.34718) 123 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:10.239215+0000 mgr.x (mgr.34718) 124 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:10.239376+0000 mgr.x (mgr.34718) 125 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:10.360603+0000 mgr.x (mgr.34718) 126 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:09:13.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:12.240172+0000 mgr.x (mgr.34718) 127 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:13.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:12.484792+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:12.485361+0000 mgr.x (mgr.34718) 128 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:09:13.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:12.485426+0000 mgr.x (mgr.34718) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:12.487143+0000 mon.b (mon.2) 155 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:12.493775+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:12.494717+0000 mgr.x (mgr.34718) 130 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:12.495226+0000 mgr.x (mgr.34718) 131 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:12.497763+0000 mgr.x (mgr.34718) 132 : cephadm [INF] Failing over to other MGR 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:12.498067+0000 mon.b (mon.2) 156 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:12.498367+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:12.504399+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:12.240172+0000 mgr.x (mgr.34718) 127 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:13.529 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:12.484792+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:12.485361+0000 mgr.x (mgr.34718) 128 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:12.485426+0000 mgr.x (mgr.34718) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:12.487143+0000 mon.b (mon.2) 155 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:12.493775+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:12.494717+0000 mgr.x (mgr.34718) 130 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:12.495226+0000 mgr.x (mgr.34718) 131 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:12.497763+0000 mgr.x (mgr.34718) 132 : cephadm [INF] Failing over to other MGR 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:12.498067+0000 mon.b (mon.2) 156 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:12.498367+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:12.504399+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:09:13.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.495+0000 7f6f6d21b640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-17T09:09:13.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ignoring --setuser ceph since I am not root 2024-09-17T09:09:13.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: ignoring --setgroup ceph since I am not root 2024-09-17T09:09:13.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.550+0000 7fe164a19640 1 -- 172.21.15.105:0/2494913026 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fe57204d00 con 0x55fe571d9400 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.643+0000 7fe167274040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.699+0000 7fe167274040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:12.240172+0000 mgr.x (mgr.34718) 127 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:12.484792+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:12.485361+0000 mgr.x (mgr.34718) 128 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:12.485426+0000 mgr.x (mgr.34718) 129 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:12.487143+0000 mon.b (mon.2) 155 : audit [DBG] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:12.493775+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.34718 ' entity='mgr.x' 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:12.494717+0000 mgr.x (mgr.34718) 130 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:12.495226+0000 mgr.x (mgr.34718) 131 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:12.497763+0000 mgr.x (mgr.34718) 132 : cephadm [INF] Failing over to other MGR 2024-09-17T09:09:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:12.498067+0000 mon.b (mon.2) 156 : audit [INF] from='mgr.34718 172.21.15.105:0/2821672655' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:12.498367+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-17T09:09:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:12.504399+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:09:14.157 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.845+0000 7fe167274040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:09:14.157 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:13.911+0000 7fe167274040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:09:14.157 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:14.056+0000 7fe167274040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:09:14.494 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:14.197+0000 7fe167274040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:09:14.533 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:09:14.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.495162+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T09:09:14.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:13.495241+0000 mon.a (mon.0) 831 : cluster [DBG] mgrmap e31: y(active, starting, since 0.996106s) 2024-09-17T09:09:14.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.497344+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.497566+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.497721+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.498250+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.498532+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.498926+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.499270+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.499642+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.500003+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.500352+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.500710+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:09:14.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.501055+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.501399+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.501673+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.495162+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:13.495241+0000 mon.a (mon.0) 831 : cluster [DBG] mgrmap e31: y(active, starting, since 0.996106s) 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.497344+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.497566+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.497721+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:09:14.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.498250+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.498532+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.498926+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.499270+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.499642+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.500003+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.500352+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.500710+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.501055+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.501399+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.501673+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:09:14.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.502458+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:13.885505+0000 mon.a (mon.0) 832 : cluster [INF] Manager daemon y is now available 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.904107+0000 mon.c (mon.1) 89 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.928517+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.929126+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.940873+0000 mon.c (mon.1) 91 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:13.941254+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.502458+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:13.885505+0000 mon.a (mon.0) 832 : cluster [INF] Manager daemon y is now available 2024-09-17T09:09:14.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.904107+0000 mon.c (mon.1) 89 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:14.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.928517+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.929126+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.940873+0000 mon.c (mon.1) 91 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:14.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:13.941254+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:14.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:14.534+0000 7fe167274040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:09:14.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:14.599+0000 7fe167274040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:09:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.495162+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.34718 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-17T09:09:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:13.495241+0000 mon.a (mon.0) 831 : cluster [DBG] mgrmap e31: y(active, starting, since 0.996106s) 2024-09-17T09:09:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.497344+0000 mon.c (mon.1) 74 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.497566+0000 mon.c (mon.1) 75 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.497721+0000 mon.c (mon.1) 76 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.498250+0000 mon.c (mon.1) 77 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.498532+0000 mon.c (mon.1) 78 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.498926+0000 mon.c (mon.1) 79 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.499270+0000 mon.c (mon.1) 80 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.499642+0000 mon.c (mon.1) 81 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.500003+0000 mon.c (mon.1) 82 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.500352+0000 mon.c (mon.1) 83 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.500710+0000 mon.c (mon.1) 84 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.501055+0000 mon.c (mon.1) 85 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:09:14.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.501399+0000 mon.c (mon.1) 86 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.501673+0000 mon.c (mon.1) 87 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.502458+0000 mon.c (mon.1) 88 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:13.885505+0000 mon.a (mon.0) 832 : cluster [INF] Manager daemon y is now available 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.904107+0000 mon.c (mon.1) 89 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.928517+0000 mon.c (mon.1) 90 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.929126+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.940873+0000 mon.c (mon.1) 91 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:13.941254+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:09:15.093 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:14.798+0000 7fe167274040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (12m) 3m ago 20m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (12m) 114s ago 20m 35.3M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 114s ago 19m - - 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443 running (15m) 114s ago 24m 477M - 18.2.4-855-g19e2a2be 980a0bfcd684 e8e063c84cda 2024-09-17T09:09:15.226 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (9m) 3m ago 25m 214M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (25m) 3m ago 25m 79.5M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (24m) 114s ago 24m 68.3M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (24m) 3m ago 24m 54.5M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 3m ago 21m 15.0M - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (12m) 114s ago 20m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (24m) 3m ago 24m 48.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (23m) 3m ago 23m 49.3M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (23m) 3m ago 23m 44.3M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (23m) 3m ago 23m 48.5M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (22m) 114s ago 22m 51.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:09:15.227 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (22m) 114s ago 22m 45.9M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:09:15.228 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (22m) 114s ago 22m 48.0M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:09:15.228 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (21m) 114s ago 21m 52.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:09:15.228 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 error 114s ago 21m - - 2024-09-17T09:09:15.228 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (19m) 3m ago 19m 40.2M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:09:15.228 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (19m) 114s ago 19m 40.9M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:09:15.639 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:09:15.639 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:09:15.639 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:09:15.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:15.641 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:09:15.641 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:09:15.641 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T09:09:15.641 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:09:15.641 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:14.508243+0000 mon.a (mon.0) 835 : cluster [DBG] mgrmap e32: y(active, since 2s) 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:14.513283+0000 mgr.y (mgr.25024) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:14.514564+0000 mgr.y (mgr.25024) 2 : audit [DBG] from='client.25153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:14.508243+0000 mon.a (mon.0) 835 : cluster [DBG] mgrmap e32: y(active, since 2s) 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:14.513283+0000 mgr.y (mgr.25024) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:15.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:14.514564+0000 mgr.y (mgr.25024) 2 : audit [DBG] from='client.25153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:09:15.459+0000 7fca289a5640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:09:09:15] ENGINE Error in HTTPServer.serve 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: Traceback (most recent call last): 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._connections.run(self.expiration_interval) 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._run(expiration_interval) 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:09:15.759 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s = self.context.wrap_socket( 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: return self.sslsocket_class._create( 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self.do_handshake() 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._sslobj.do_handshake() 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:09:15.760 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:09:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:09:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:14.508243+0000 mon.a (mon.0) 835 : cluster [DBG] mgrmap e32: y(active, since 2s) 2024-09-17T09:09:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:14.513283+0000 mgr.y (mgr.25024) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:14.514564+0000 mgr.y (mgr.25024) 2 : audit [DBG] from='client.25153 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:15.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:15.718+0000 7fe167274040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:09:15.990 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:09:16.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:15.786+0000 7fe167274040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:09:16.765 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:15.336660+0000 mgr.y (mgr.25024) 5 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTING 2024-09-17T09:09:16.765 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:15.456192+0000 mgr.y (mgr.25024) 6 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:09:16.765 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:15.459708+0000 mgr.y (mgr.25024) 7 : cephadm [ERR] [17/Sep/2024:09:09:15] ENGINE Error in HTTPServer.serve 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: Traceback (most recent call last): 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._connections.run(self.expiration_interval) 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._run(expiration_interval) 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:09:16.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: s = self.context.wrap_socket( 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: return self.sslsocket_class._create( 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self.do_handshake() 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: self._sslobj.do_handshake() 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:15.499836+0000 mgr.y (mgr.25024) 8 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:15.503785+0000 mon.a (mon.0) 836 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:09:16.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:15.503809+0000 mon.a (mon.0) 837 : cluster [INF] Cluster is now healthy 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:15.515587+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:15.557509+0000 mgr.y (mgr.25024) 9 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:15.557623+0000 mgr.y (mgr.25024) 10 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTED 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:15.639517+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.27:0/1729994841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:16.565+0000 7fe167274040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:09:16.768 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:16.703+0000 7fe167274040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:15.336660+0000 mgr.y (mgr.25024) 5 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTING 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:15.456192+0000 mgr.y (mgr.25024) 6 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:15.459708+0000 mgr.y (mgr.25024) 7 : cephadm [ERR] [17/Sep/2024:09:09:15] ENGINE Error in HTTPServer.serve 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: Traceback (most recent call last): 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:09:17.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._connections.run(self.expiration_interval) 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._run(expiration_interval) 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:09:17.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s = self.context.wrap_socket( 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: return self.sslsocket_class._create( 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self.do_handshake() 2024-09-17T09:09:17.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._sslobj.do_handshake() 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:15.499836+0000 mgr.y (mgr.25024) 8 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:15.503785+0000 mon.a (mon.0) 836 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:15.503809+0000 mon.a (mon.0) 837 : cluster [INF] Cluster is now healthy 2024-09-17T09:09:17.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:15.515587+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-17T09:09:17.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:15.557509+0000 mgr.y (mgr.25024) 9 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:09:17.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:15.557623+0000 mgr.y (mgr.25024) 10 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTED 2024-09-17T09:09:17.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:15.639517+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.27:0/1729994841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:15.336660+0000 mgr.y (mgr.25024) 5 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTING 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:15.456192+0000 mgr.y (mgr.25024) 6 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:15.459708+0000 mgr.y (mgr.25024) 7 : cephadm [ERR] [17/Sep/2024:09:09:15] ENGINE Error in HTTPServer.serve 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: Traceback (most recent call last): 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._connections.run(self.expiration_interval) 2024-09-17T09:09:17.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._run(expiration_interval) 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:09:17.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: s = self.context.wrap_socket( 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: return self.sslsocket_class._create( 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self.do_handshake() 2024-09-17T09:09:17.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: self._sslobj.do_handshake() 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:15.499836+0000 mgr.y (mgr.25024) 8 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:15.503785+0000 mon.a (mon.0) 836 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 2 failed cephadm daemon(s)) 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:15.503809+0000 mon.a (mon.0) 837 : cluster [INF] Cluster is now healthy 2024-09-17T09:09:17.017 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:15.515587+0000 mon.a (mon.0) 838 : cluster [DBG] mgrmap e33: y(active, since 3s) 2024-09-17T09:09:17.018 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:15.557509+0000 mgr.y (mgr.25024) 9 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:09:17.018 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:15.557623+0000 mgr.y (mgr.25024) 10 : cephadm [INF] [17/Sep/2024:09:09:15] ENGINE Bus STARTED 2024-09-17T09:09:17.018 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:15.639517+0000 mon.a (mon.0) 839 : audit [DBG] from='client.? 172.21.15.27:0/1729994841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:17.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:16.881+0000 7fe167274040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:09:17.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:16.952+0000 7fe167274040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:09:17.463 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.135+0000 7fe167274040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:09:17.464 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.210+0000 7fe167274040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:09:17.464 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.274+0000 7fe167274040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:09:17.464 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.462+0000 7fe167274040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:09:17.757 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:15.989481+0000 mgr.y (mgr.25024) 11 : audit [DBG] from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:17.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:15.989481+0000 mgr.y (mgr.25024) 11 : audit [DBG] from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:17.763 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:15.989481+0000 mgr.y (mgr.25024) 11 : audit [DBG] from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:17.763 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.540+0000 7fe167274040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:09:17.763 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.620+0000 7fe167274040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:09:17.763 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.703+0000 7fe167274040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:09:17.763 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.762+0000 7fe167274040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:09:18.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:17.820+0000 7fe167274040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:09:18.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x[40808]: debug 2024-09-17T09:09:18.636+0000 7fe167274040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:09:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:17.500252+0000 mgr.y (mgr.25024) 12 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:17.522012+0000 mon.a (mon.0) 840 : cluster [DBG] mgrmap e34: y(active, since 5s) 2024-09-17T09:09:19.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:17.500252+0000 mgr.y (mgr.25024) 12 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:17.522012+0000 mon.a (mon.0) 840 : cluster [DBG] mgrmap e34: y(active, since 5s) 2024-09-17T09:09:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:17.500252+0000 mgr.y (mgr.25024) 12 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:17.522012+0000 mon.a (mon.0) 840 : cluster [DBG] mgrmap e34: y(active, since 5s) 2024-09-17T09:09:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:18.639140+0000 mon.a (mon.0) 841 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.640142+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.640697+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.641352+0000 mon.b (mon.2) 159 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.641580+0000 mon.b (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.771688+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:18.776309+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.224044+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.232627+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.309164+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:19.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.316047+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:18.639140+0000 mon.a (mon.0) 841 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.640142+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.640697+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.641352+0000 mon.b (mon.2) 159 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.641580+0000 mon.b (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.771688+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:18.776309+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.224044+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.232627+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.309164+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.316047+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:18.639140+0000 mon.a (mon.0) 841 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.640142+0000 mon.b (mon.2) 157 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.640697+0000 mon.b (mon.2) 158 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.641352+0000 mon.b (mon.2) 159 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.641580+0000 mon.b (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.105:0/534835314' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.771688+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:18.776309+0000 mon.a (mon.0) 843 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.224044+0000 mon.a (mon.0) 844 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.232627+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.309164+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.316047+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:19.500596+0000 mgr.y (mgr.25024) 13 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.534864+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:19.535711+0000 mon.a (mon.0) 848 : cluster [DBG] mgrmap e35: y(active, since 7s), standbys: x 2024-09-17T09:09:20.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.736647+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.744972+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.746633+0000 mon.c (mon.1) 93 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.747312+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:19.747931+0000 mgr.y (mgr.25024) 14 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:19.748704+0000 mgr.y (mgr.25024) 15 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:19.755429+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:20.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:20.314648+0000 mon.a (mon.0) 852 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:19.500596+0000 mgr.y (mgr.25024) 13 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.534864+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:19.535711+0000 mon.a (mon.0) 848 : cluster [DBG] mgrmap e35: y(active, since 7s), standbys: x 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.736647+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.744972+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.746633+0000 mon.c (mon.1) 93 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.747312+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:21.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:19.747931+0000 mgr.y (mgr.25024) 14 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:19.748704+0000 mgr.y (mgr.25024) 15 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:19.755429+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:20.314648+0000 mon.a (mon.0) 852 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:19.500596+0000 mgr.y (mgr.25024) 13 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 87 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.534864+0000 mon.c (mon.1) 92 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:19.535711+0000 mon.a (mon.0) 848 : cluster [DBG] mgrmap e35: y(active, since 7s), standbys: x 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.736647+0000 mon.a (mon.0) 849 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.744972+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.746633+0000 mon.c (mon.1) 93 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.747312+0000 mon.c (mon.1) 94 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:21.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:19.747931+0000 mgr.y (mgr.25024) 14 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:21.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:19.748704+0000 mgr.y (mgr.25024) 15 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:09:21.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:19.755429+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:21.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:20.314648+0000 mon.a (mon.0) 852 : cluster [WRN] Health check failed: 2 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:09:21.866 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:19.825942+0000 mgr.y (mgr.25024) 16 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:09:21.866 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:20.080346+0000 mgr.y (mgr.25024) 17 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:09:21.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:19.825942+0000 mgr.y (mgr.25024) 16 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:09:21.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:20.080346+0000 mgr.y (mgr.25024) 17 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:09:21.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:19.825942+0000 mgr.y (mgr.25024) 16 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-17T09:09:21.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:20.080346+0000 mgr.y (mgr.25024) 17 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi105 2024-09-17T09:09:22.925 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:21.749274+0000 mgr.y (mgr.25024) 18 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 115 op/s 2024-09-17T09:09:22.926 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:21.749274+0000 mgr.y (mgr.25024) 18 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 115 op/s 2024-09-17T09:09:22.990 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:21.749274+0000 mgr.y (mgr.25024) 18 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 115 op/s 2024-09-17T09:09:23.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:23.567 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 podman[59130]: 2024-09-17 09:09:23.205048037 +0000 UTC m=+0.016967971 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:09:23.567 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 podman[59130]: 2024-09-17 09:09:23.308709354 +0000 UTC m=+0.120629287 container create 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:23.567 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 podman[59130]: 2024-09-17 09:09:23.535413383 +0000 UTC m=+0.347333317 container init 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:23.567 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 podman[59130]: 2024-09-17 09:09:23.538279846 +0000 UTC m=+0.350199778 container start 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:23.923 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.567Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.568Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.568Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.571Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:09:23.924 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.571Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.579Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.579Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.579Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.579Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.580Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.580Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.580Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.580Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.580Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.581Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=17 2024-09-17T09:09:23.925 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.581Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.582Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.582Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.583Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.583Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.583Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.583Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=17 maxSegment=17 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.583Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=12.481914ms 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:09:23.926 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=info ts=2024-09-17T09:09:23.585Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[59145]: level=error ts=2024-09-17T09:09:23.586Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 bash[59130]: 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 2024-09-17T09:09:23.927 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:23.928 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:23 smithi105 podman[59162]: 2024-09-17 09:09:23.642761416 +0000 UTC m=+0.049845831 container died 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:24.447 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:24 smithi105 podman[59162]: 2024-09-17 09:09:24.192147331 +0000 UTC m=+0.599231779 container remove 3ae5244582fdec7d9b58d0d4fa32a95c358f24900488ef65c945c7253ae92fb2 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:24.447 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:24 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:09:24.447 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:24 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:09:24.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:23.663327+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:23.672954+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:23.674385+0000 mgr.y (mgr.25024) 19 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:23.674668+0000 mon.c (mon.1) 95 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:23.675002+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:23.678731+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:23.679647+0000 mgr.y (mgr.25024) 20 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:09:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:23.749677+0000 mgr.y (mgr.25024) 21 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 90 op/s 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:23.663327+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:23.672954+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:23.674385+0000 mgr.y (mgr.25024) 19 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:23.674668+0000 mon.c (mon.1) 95 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:23.675002+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:23.678731+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:23.679647+0000 mgr.y (mgr.25024) 20 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:09:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:23.749677+0000 mgr.y (mgr.25024) 21 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 90 op/s 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:23.663327+0000 mon.a (mon.0) 853 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:23.672954+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:23.674385+0000 mgr.y (mgr.25024) 19 : cephadm [INF] Reconfiguring iscsi.foo.smithi105.mzuaqj (dependencies changed)... 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:23.674668+0000 mon.c (mon.1) 95 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:23.675002+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:23.678731+0000 mon.c (mon.1) 96 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:23.679647+0000 mgr.y (mgr.25024) 20 : cephadm [INF] Reconfiguring daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:09:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:23.749677+0000 mgr.y (mgr.25024) 21 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 90 op/s 2024-09-17T09:09:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:25.750492+0000 mgr.y (mgr.25024) 22 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:26.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:25.750492+0000 mgr.y (mgr.25024) 22 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:26.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:25.750492+0000 mgr.y (mgr.25024) 22 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.433939+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.440044+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.445635+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.446204+0000 mgr.y (mgr.25024) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.450136+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.450688+0000 mgr.y (mgr.25024) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.454235+0000 mon.c (mon.1) 99 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.454657+0000 mgr.y (mgr.25024) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.494598+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:26.909996+0000 mon.b (mon.2) 161 : audit [DBG] from='client.? 172.21.15.105:0/3711541086' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:27.238463+0000 mon.b (mon.2) 162 : audit [INF] from='client.? 172.21.15.105:0/3609695100' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:27.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.433939+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.440044+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.445635+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.446204+0000 mgr.y (mgr.25024) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.450136+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.450688+0000 mgr.y (mgr.25024) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.454235+0000 mon.c (mon.1) 99 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.454657+0000 mgr.y (mgr.25024) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.494598+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:26.909996+0000 mon.b (mon.2) 161 : audit [DBG] from='client.? 172.21.15.105:0/3711541086' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:27.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:27.238463+0000 mon.b (mon.2) 162 : audit [INF] from='client.? 172.21.15.105:0/3609695100' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.433939+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.440044+0000 mon.a (mon.0) 857 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.445635+0000 mon.c (mon.1) 97 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.446204+0000 mgr.y (mgr.25024) 23 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.450136+0000 mon.c (mon.1) 98 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.450688+0000 mgr.y (mgr.25024) 24 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:09:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.454235+0000 mon.c (mon.1) 99 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.454657+0000 mgr.y (mgr.25024) 25 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:09:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.494598+0000 mon.c (mon.1) 100 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:26.909996+0000 mon.b (mon.2) 161 : audit [DBG] from='client.? 172.21.15.105:0/3711541086' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:27.238463+0000 mon.b (mon.2) 162 : audit [INF] from='client.? 172.21.15.105:0/3609695100' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:28.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:27.750871+0000 mgr.y (mgr.25024) 26 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:28.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:27.750871+0000 mgr.y (mgr.25024) 26 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:28.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:27.750871+0000 mgr.y (mgr.25024) 26 : cluster [DBG] pgmap v11: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:29.249857+0000 mon.c (mon.1) 101 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:29.250444+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:29.251607+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:09:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:29.252608+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:29.249857+0000 mon.c (mon.1) 101 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:29.250444+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:29.251607+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:29.252608+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:29.249857+0000 mon.c (mon.1) 101 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:29.250444+0000 mon.a (mon.0) 858 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:09:29.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:29.251607+0000 mon.c (mon.1) 102 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:09:29.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:29.252608+0000 mon.c (mon.1) 103 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:30.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:29.249371+0000 mgr.y (mgr.25024) 27 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-17T09:09:30.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:29.253604+0000 mgr.y (mgr.25024) 28 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T09:09:30.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:29.751772+0000 mgr.y (mgr.25024) 29 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:29.249371+0000 mgr.y (mgr.25024) 27 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:29.253604+0000 mgr.y (mgr.25024) 28 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:29.751772+0000 mgr.y (mgr.25024) 29 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:29.249371+0000 mgr.y (mgr.25024) 27 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:29.253604+0000 mgr.y (mgr.25024) 28 : cephadm [INF] Deploying daemon mgr.x on smithi105 2024-09-17T09:09:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:29.751772+0000 mgr.y (mgr.25024) 29 : cluster [DBG] pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 0 B/s wr, 151 op/s 2024-09-17T09:09:31.274 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:30 smithi105 systemd[1]: Stopping Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:31.275 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 podman[59845]: 2024-09-17 09:09:31.038560563 +0000 UTC m=+0.129250373 container died e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:09:31.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 podman[59845]: 2024-09-17 09:09:31.407254362 +0000 UTC m=+0.497944160 container cleanup e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:09:31.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 bash[59845]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x 2024-09-17T09:09:31.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:31.752226+0000 mgr.y (mgr.25024) 30 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:31.931 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 podman[59857]: 2024-09-17 09:09:31.675780334 +0000 UTC m=+0.638498463 container remove e8e063c84cdaa7dec1ad1996734596ec1880415aaf1ae4b13dfb45d2e5645a27 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr.x, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True) 2024-09-17T09:09:31.931 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:09:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:31.752226+0000 mgr.y (mgr.25024) 30 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:31.752226+0000 mgr.y (mgr.25024) 30 : cluster [DBG] pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:32.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-17T09:09:32.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:31 smithi105 systemd[1]: Stopped Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:32.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 systemd[1]: Starting Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:32.654 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 podman[59998]: 2024-09-17 09:09:32.249675715 +0000 UTC m=+0.015487555 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:09:32.654 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 podman[59998]: 2024-09-17 09:09:32.38500146 +0000 UTC m=+0.150813296 container create ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T09:09:33.005 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 podman[59998]: 2024-09-17 09:09:32.654098874 +0000 UTC m=+0.419910719 container init ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 podman[59998]: 2024-09-17 09:09:32.656842522 +0000 UTC m=+0.422654367 container start ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 ceph-mgr[60017]: -- 172.21.15.105:0/436923211 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5560c5716d00 con 0x5560c56eb400 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 bash[59998]: ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 systemd[1]: Started Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:32.781+0000 7ff6a50ad040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:32.844+0000 7ff6a50ad040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:09:33.006 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.003+0000 7ff6a50ad040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:09:33.277 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.078+0000 7ff6a50ad040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:09:33.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.240+0000 7ff6a50ad040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:09:33.716 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.388+0000 7ff6a50ad040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:09:33.717 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.714+0000 7ff6a50ad040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:09:33.973 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.775+0000 7ff6a50ad040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:09:33.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:32.814490+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:33.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:32.820416+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:33.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:32.821962+0000 mon.c (mon.1) 104 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:33.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:33.752849+0000 mgr.y (mgr.25024) 31 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:32.814490+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:32.820416+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:32.821962+0000 mon.c (mon.1) 104 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:33.752849+0000 mgr.y (mgr.25024) 31 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:34.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:32.814490+0000 mon.a (mon.0) 859 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:34.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:32.820416+0000 mon.a (mon.0) 860 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:34.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:32.821962+0000 mon.c (mon.1) 104 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:34.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:33.752849+0000 mgr.y (mgr.25024) 31 : cluster [DBG] pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:34.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:33.971+0000 7ff6a50ad040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:09:34.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:34 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 1. 2024-09-17T09:09:34.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:34 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:34.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:34 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:35.072 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:34.897+0000 7ff6a50ad040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:09:35.072 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:34.967+0000 7ff6a50ad040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:09:35.072 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:34 smithi105 podman[60352]: 2024-09-17 09:09:34.711915023 +0000 UTC m=+0.025789980 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:09:35.073 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:34 smithi105 podman[60352]: 2024-09-17 09:09:34.828760192 +0000 UTC m=+0.142635119 container create aa2b8ec84ab165b7045cecb6a3cabd8d77987352b870913bb9ea056eca4ad418 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:35.073 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 podman[60352]: 2024-09-17 09:09:35.072147181 +0000 UTC m=+0.386022114 container init aa2b8ec84ab165b7045cecb6a3cabd8d77987352b870913bb9ea056eca4ad418 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:35.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 podman[60352]: 2024-09-17 09:09:35.075050064 +0000 UTC m=+0.388924998 container start aa2b8ec84ab165b7045cecb6a3cabd8d77987352b870913bb9ea056eca4ad418 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.104Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.106Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.106Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.108Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.108Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=18 2024-09-17T09:09:35.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.117Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.117Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.118Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.119Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.120Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.120Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=18 2024-09-17T09:09:35.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.120Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=17 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=18 maxSegment=18 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.121Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=13.532018ms 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.123Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.123Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.123Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:09:35.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=info ts=2024-09-17T09:09:35.124Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[60366]: level=error ts=2024-09-17T09:09:35.124Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 bash[60352]: aa2b8ec84ab165b7045cecb6a3cabd8d77987352b870913bb9ea056eca4ad418 2024-09-17T09:09:35.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:35.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:35.753612+0000 mgr.y (mgr.25024) 32 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:35.998 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 podman[60384]: 2024-09-17 09:09:35.889419227 +0000 UTC m=+0.756646466 container remove aa2b8ec84ab165b7045cecb6a3cabd8d77987352b870913bb9ea056eca4ad418 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:35.998 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:09:35.998 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:35 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:09:35.998 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:35.727+0000 7ff6a50ad040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:09:35.998 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:35.844+0000 7ff6a50ad040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:09:35.998 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:35.995+0000 7ff6a50ad040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:09:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:35.753612+0000 mgr.y (mgr.25024) 32 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:35.753612+0000 mgr.y (mgr.25024) 32 : cluster [DBG] pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 0 B/s wr, 59 op/s 2024-09-17T09:09:36.277 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.060+0000 7ff6a50ad040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:09:36.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.221+0000 7ff6a50ad040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:09:36.608 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.303+0000 7ff6a50ad040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:09:36.608 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.367+0000 7ff6a50ad040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:09:36.608 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.537+0000 7ff6a50ad040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:09:36.608 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.606+0000 7ff6a50ad040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:09:36.884 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.681+0000 7ff6a50ad040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:09:36.884 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.766+0000 7ff6a50ad040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:09:36.884 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.825+0000 7ff6a50ad040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:09:37.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:36.886+0000 7ff6a50ad040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:09:38.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:09:37.710+0000 7ff6a50ad040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:09:38.106 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:37.714097+0000 mon.a (mon.0) 861 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T09:09:38.106 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:37.714241+0000 mon.a (mon.0) 862 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:38.107 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:37.715352+0000 mon.b (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:38.107 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:37.715890+0000 mon.b (mon.2) 164 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:38.107 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:37.716656+0000 mon.b (mon.2) 165 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:38.107 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:37.716926+0000 mon.b (mon.2) 166 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:38.107 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:37.754091+0000 mgr.y (mgr.25024) 33 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:37.714097+0000 mon.a (mon.0) 861 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:37.714241+0000 mon.a (mon.0) 862 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:37.715352+0000 mon.b (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:37.715890+0000 mon.b (mon.2) 164 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:37.716656+0000 mon.b (mon.2) 165 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:37.716926+0000 mon.b (mon.2) 166 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:37.754091+0000 mgr.y (mgr.25024) 33 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:37.714097+0000 mon.a (mon.0) 861 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:37.714241+0000 mon.a (mon.0) 862 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:37.715352+0000 mon.b (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:37.715890+0000 mon.b (mon.2) 164 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:37.716656+0000 mon.b (mon.2) 165 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:37.716926+0000 mon.b (mon.2) 166 : audit [DBG] from='mgr.? 172.21.15.105:0/2910543699' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:09:38.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:37.754091+0000 mgr.y (mgr.25024) 33 : cluster [DBG] pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:39.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:37.774193+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e36: y(active, since 25s), standbys: x 2024-09-17T09:09:39.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:38.525042+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:39.043 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:38.533567+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:37.774193+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e36: y(active, since 25s), standbys: x 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:38.525042+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:38.533567+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:37.774193+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e36: y(active, since 25s), standbys: x 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:38.525042+0000 mon.a (mon.0) 864 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:38.533567+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:39.168404+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:39.175722+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.352 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:39.754861+0000 mgr.y (mgr.25024) 34 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:39.168404+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:39.175722+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:39.754861+0000 mgr.y (mgr.25024) 34 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:39.168404+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:39.175722+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:40.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:39.754861+0000 mgr.y (mgr.25024) 34 : cluster [DBG] pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:41.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:40.637134+0000 mon.b (mon.2) 167 : audit [DBG] from='client.? 172.21.15.105:0/3324254522' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:41.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:40.968942+0000 mon.b (mon.2) 168 : audit [INF] from='client.? 172.21.15.105:0/2201403370' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:41.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:40.637134+0000 mon.b (mon.2) 167 : audit [DBG] from='client.? 172.21.15.105:0/3324254522' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:41.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:40.968942+0000 mon.b (mon.2) 168 : audit [INF] from='client.? 172.21.15.105:0/2201403370' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:41.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:40.637134+0000 mon.b (mon.2) 167 : audit [DBG] from='client.? 172.21.15.105:0/3324254522' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:41.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:40.968942+0000 mon.b (mon.2) 168 : audit [INF] from='client.? 172.21.15.105:0/2201403370' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:42.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:41.755330+0000 mgr.y (mgr.25024) 35 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:42.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:41.755330+0000 mgr.y (mgr.25024) 35 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:42.524 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:41.755330+0000 mgr.y (mgr.25024) 35 : cluster [DBG] pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.382509+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.389060+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.390591+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:44.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.391532+0000 mon.c (mon.1) 106 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:44.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.392297+0000 mgr.y (mgr.25024) 36 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:44.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:43.392865+0000 mgr.y (mgr.25024) 37 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:43.393049+0000 mgr.y (mgr.25024) 38 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.397572+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.517616+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.520485+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.521155+0000 mgr.y (mgr.25024) 39 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:09:44.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.525545+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.526982+0000 mon.c (mon.1) 109 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.527250+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.531884+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T09:09:44.512 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.533155+0000 mon.c (mon.1) 110 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.382509+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.389060+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.390591+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:44.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.391532+0000 mon.c (mon.1) 106 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.392297+0000 mgr.y (mgr.25024) 36 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:43.392865+0000 mgr.y (mgr.25024) 37 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:43.393049+0000 mgr.y (mgr.25024) 38 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.397572+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.517616+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:44.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.520485+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.521155+0000 mgr.y (mgr.25024) 39 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:09:44.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.525545+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.526982+0000 mon.c (mon.1) 109 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.527250+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.531884+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T09:09:44.516 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.533155+0000 mon.c (mon.1) 110 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.533404+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.536704+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T09:09:44.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.538704+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.539349+0000 mgr.y (mgr.25024) 40 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-17T09:09:44.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.543520+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.545649+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.546201+0000 mgr.y (mgr.25024) 41 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.549838+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.551852+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.552444+0000 mgr.y (mgr.25024) 42 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.556081+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.557942+0000 mon.c (mon.1) 114 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.558547+0000 mgr.y (mgr.25024) 43 : cephadm [INF] Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:09:44.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.562169+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.564023+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.564660+0000 mgr.y (mgr.25024) 44 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:09:44.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.568414+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.570265+0000 mon.c (mon.1) 116 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.570804+0000 mgr.y (mgr.25024) 45 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.533404+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.536704+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.538704+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.539349+0000 mgr.y (mgr.25024) 40 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.543520+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.545649+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.546201+0000 mgr.y (mgr.25024) 41 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.549838+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.551852+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.552444+0000 mgr.y (mgr.25024) 42 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.556081+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.557942+0000 mon.c (mon.1) 114 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.558547+0000 mgr.y (mgr.25024) 43 : cephadm [INF] Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.562169+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.564023+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.564660+0000 mgr.y (mgr.25024) 44 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.568414+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.570265+0000 mon.c (mon.1) 116 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.570804+0000 mgr.y (mgr.25024) 45 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.574357+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.578847+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:43.579439+0000 mgr.y (mgr.25024) 46 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:43.583021+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.574357+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.578847+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:43.579439+0000 mgr.y (mgr.25024) 46 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-17T09:09:44.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:43.583021+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.382509+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.389060+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.390591+0000 mon.c (mon.1) 105 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.391532+0000 mon.c (mon.1) 106 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.392297+0000 mgr.y (mgr.25024) 36 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:43.392865+0000 mgr.y (mgr.25024) 37 : cluster [DBG] pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:43.393049+0000 mgr.y (mgr.25024) 38 : cluster [DBG] pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.397572+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.517616+0000 mon.c (mon.1) 107 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.520485+0000 mon.c (mon.1) 108 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.521155+0000 mgr.y (mgr.25024) 39 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.525545+0000 mon.a (mon.0) 871 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.526982+0000 mon.c (mon.1) 109 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.527250+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-17T09:09:44.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.531884+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.533155+0000 mon.c (mon.1) 110 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.533404+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.536704+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.538704+0000 mon.c (mon.1) 111 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.539349+0000 mgr.y (mgr.25024) 40 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.543520+0000 mon.a (mon.0) 876 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.545649+0000 mon.c (mon.1) 112 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.546201+0000 mgr.y (mgr.25024) 41 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.549838+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.551852+0000 mon.c (mon.1) 113 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.552444+0000 mgr.y (mgr.25024) 42 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.556081+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.557942+0000 mon.c (mon.1) 114 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.558547+0000 mgr.y (mgr.25024) 43 : cephadm [INF] Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.562169+0000 mon.a (mon.0) 879 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.564023+0000 mon.c (mon.1) 115 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.564660+0000 mgr.y (mgr.25024) 44 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.568414+0000 mon.a (mon.0) 880 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.570265+0000 mon.c (mon.1) 116 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.570804+0000 mgr.y (mgr.25024) 45 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-17T09:09:44.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.574357+0000 mon.a (mon.0) 881 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:44.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.578847+0000 mon.c (mon.1) 117 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:44.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:43.579439+0000 mgr.y (mgr.25024) 46 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-17T09:09:44.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:43.583021+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:45.446 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:45 smithi027 systemd[1]: Stopping Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:45.757 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:45 smithi027 podman[60563]: 2024-09-17 09:09:45.455556457 +0000 UTC m=+0.110450325 container died cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.052 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:45 smithi027 podman[60563]: 2024-09-17 09:09:45.761337663 +0000 UTC m=+0.416231530 container cleanup cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.052 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:45 smithi027 bash[60563]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a 2024-09-17T09:09:46.053 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 podman[60575]: 2024-09-17 09:09:46.004244863 +0000 UTC m=+0.546533380 container remove cbf4562ed79b42890f76afae101ec745bdc8fec81ba1f5c61a0e582aae6e65b0 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.053 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:09:46.324 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 2. 2024-09-17T09:09:46.324 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:46.324 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:46.324 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 podman[61483]: 2024-09-17 09:09:46.118643078 +0000 UTC m=+0.018373571 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:09:46.324 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 podman[61483]: 2024-09-17 09:09:46.230137733 +0000 UTC m=+0.129868216 container create a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.367 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:09:46.721 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-17T09:09:46.722 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 systemd[1]: Stopped Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:46.722 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 systemd[1]: Starting Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:46.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:44.840145+0000 mgr.y (mgr.25024) 47 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-17T09:09:46.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:44.840452+0000 mgr.y (mgr.25024) 48 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T09:09:46.722 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:45.393704+0000 mgr.y (mgr.25024) 49 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:46.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:44.840145+0000 mgr.y (mgr.25024) 47 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-17T09:09:46.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:44.840452+0000 mgr.y (mgr.25024) 48 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T09:09:46.723 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:45.393704+0000 mgr.y (mgr.25024) 49 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:44.840145+0000 mgr.y (mgr.25024) 47 : cephadm [INF] Upgrade: Updating node-exporter.a (1/2) 2024-09-17T09:09:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:44.840452+0000 mgr.y (mgr.25024) 48 : cephadm [INF] Deploying daemon node-exporter.a on smithi027 2024-09-17T09:09:46.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:45.393704+0000 mgr.y (mgr.25024) 49 : cluster [DBG] pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 podman[61483]: 2024-09-17 09:09:46.465101211 +0000 UTC m=+0.364831692 container init a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 podman[61483]: 2024-09-17 09:09:46.467921984 +0000 UTC m=+0.367652474 container start a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.497Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.499Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.499Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:09:46.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.501Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.501Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.509Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.509Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.510Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.511Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.511Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=19 2024-09-17T09:09:46.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.512Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=17 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=18 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=19 maxSegment=19 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.513Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=12.782763ms 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.515Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.515Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.515Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:09:46.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=info ts=2024-09-17T09:09:46.516Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[61498]: level=error ts=2024-09-17T09:09:46.516Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 bash[61483]: a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:46.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:46 smithi105 podman[61515]: 2024-09-17 09:09:46.558553948 +0000 UTC m=+0.034828462 container died a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:47.007 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:46 smithi027 bash[60766]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-17T09:09:47.053 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (13m) 28s ago 20m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (12m) 8s ago 20m 35.8M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 8s ago 20m - - 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (14s) 8s ago 24m 62.1M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (10m) 28s ago 26m 454M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (26m) 28s ago 26m 85.6M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (25m) 8s ago 25m 66.2M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (25m) 28s ago 25m 60.6M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (13m) 28s ago 21m 15.0M - 0.18.1 e5a616e4b9cf cbf4562ed79b 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (12m) 8s ago 21m 10.7M - 0.18.1 e5a616e4b9cf 3baf013bbfe2 2024-09-17T09:09:47.054 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (24m) 28s ago 24m 48.7M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (24m) 28s ago 24m 49.3M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (23m) 28s ago 23m 44.5M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (23m) 28s ago 23m 48.4M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (23m) 8s ago 23m 53.5M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (23m) 8s ago 23m 47.0M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (22m) 8s ago 22m 48.5M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (22m) 8s ago 22m 52.6M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 unknown 8s ago 21m - - 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (20m) 28s ago 20m 41.0M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:09:47.055 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (20m) 8s ago 20m 42.6M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:09:47.451 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:47 smithi027 bash[60766]: Getting image source signatures 2024-09-17T09:09:47.451 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:47 smithi027 bash[60766]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-17T09:09:47.451 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:47 smithi027 bash[60766]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-17T09:09:47.452 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:47 smithi027 bash[60766]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-17T09:09:47.452 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:47 smithi105 podman[61515]: 2024-09-17 09:09:47.085506142 +0000 UTC m=+0.561780643 container remove a93904644be77d296038a5be387ff45d12a96ae7afcb946a0e66b61fc2d0822a (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:47.452 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:09:47.452 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:47 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:09:47.485 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:09:47.486 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:09:47.487 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:09:47.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:46.344927+0000 mgr.y (mgr.25024) 50 : audit [DBG] from='client.25255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:46.719041+0000 mgr.y (mgr.25024) 51 : audit [DBG] from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:46.344927+0000 mgr.y (mgr.25024) 50 : audit [DBG] from='client.25255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:46.719041+0000 mgr.y (mgr.25024) 51 : audit [DBG] from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:46.344927+0000 mgr.y (mgr.25024) 50 : audit [DBG] from='client.25255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:46.719041+0000 mgr.y (mgr.25024) 51 : audit [DBG] from='client.25261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:47.879 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:09:47.879 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:09:47.880 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:09:48.758 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:48 smithi027 bash[60766]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-17T09:09:48.758 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:48 smithi027 bash[60766]: Writing manifest to image destination 2024-09-17T09:09:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:47.045687+0000 mgr.y (mgr.25024) 52 : audit [DBG] from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:47.394063+0000 mgr.y (mgr.25024) 53 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:47.485632+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.27:0/3930670410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:48.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:47.045687+0000 mgr.y (mgr.25024) 52 : audit [DBG] from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:48.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:47.394063+0000 mgr.y (mgr.25024) 53 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:48.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:47.485632+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.27:0/3930670410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:48.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:47.045687+0000 mgr.y (mgr.25024) 52 : audit [DBG] from='client.25267 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:48.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:47.394063+0000 mgr.y (mgr.25024) 53 : cluster [DBG] pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:48.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:47.485632+0000 mon.a (mon.0) 883 : audit [DBG] from='client.? 172.21.15.27:0/3930670410' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:09:49.258 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:48 smithi027 podman[60766]: 2024-09-17 09:09:48.859629232 +0000 UTC m=+2.234577334 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-17T09:09:49.258 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:48 smithi027 podman[60766]: 2024-09-17 09:09:48.949716376 +0000 UTC m=+2.324664467 container create 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 podman[60766]: 2024-09-17 09:09:49.177224024 +0000 UTC m=+2.552172120 container init 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 podman[60766]: 2024-09-17 09:09:49.180720722 +0000 UTC m=+2.555668814 container start 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.184Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.184Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=arp 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-17T09:09:49.259 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-17T09:09:49.260 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=edac 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.185Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-17T09:09:49.261 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=os 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=stat 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-17T09:09:49.262 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=time 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=uname 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-17T09:09:49.263 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a[60911]: ts=2024-09-17T09:09:49.186Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-17T09:09:49.513 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 bash[60766]: 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 2024-09-17T09:09:49.513 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:09:49 smithi027 systemd[1]: Started Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:49.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:47.878771+0000 mgr.y (mgr.25024) 54 : audit [DBG] from='client.25279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:49.513 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:49.340220+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:49.514 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:49.345698+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:49.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:47.878771+0000 mgr.y (mgr.25024) 54 : audit [DBG] from='client.25279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:49.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:49.340220+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:49.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:49.345698+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:47.878771+0000 mgr.y (mgr.25024) 54 : audit [DBG] from='client.25279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:09:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:49.340220+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:49.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:49.345698+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:50.600 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:49.394403+0000 mgr.y (mgr.25024) 55 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:50.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:49.394403+0000 mgr.y (mgr.25024) 55 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:49.394403+0000 mgr.y (mgr.25024) 55 : cluster [DBG] pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:51.213 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 systemd[1]: Stopping Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:51.469 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 podman[61786]: 2024-09-17 09:09:51.222483542 +0000 UTC m=+0.101515661 container died 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:50.622393+0000 mgr.y (mgr.25024) 56 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-17T09:09:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:09:50.622672+0000 mgr.y (mgr.25024) 57 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T09:09:51.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:50.622393+0000 mgr.y (mgr.25024) 56 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-17T09:09:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:09:50.622672+0000 mgr.y (mgr.25024) 57 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T09:09:51.778 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 podman[61786]: 2024-09-17 09:09:51.569941241 +0000 UTC m=+0.448973349 container cleanup 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:51.778 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 bash[61786]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b 2024-09-17T09:09:51.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:50.622393+0000 mgr.y (mgr.25024) 56 : cephadm [INF] Upgrade: Updating node-exporter.b (2/2) 2024-09-17T09:09:51.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:09:50.622672+0000 mgr.y (mgr.25024) 57 : cephadm [INF] Deploying daemon node-exporter.b on smithi105 2024-09-17T09:09:52.110 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 podman[61800]: 2024-09-17 09:09:51.846217955 +0000 UTC m=+0.625722374 container remove 3baf013bbfe2dc4efd24f678733192a233aa6da92bc0657f6e68a5fcf5ed3d19 (image=docker.io/prom/node-exporter:v0.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter.b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:52.110 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:51 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:09:52.110 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-17T09:09:52.110 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 systemd[1]: Stopped Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:52.469 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 systemd[1]: Starting Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:52.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:51.395264+0000 mgr.y (mgr.25024) 58 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:52.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:51.395264+0000 mgr.y (mgr.25024) 58 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:52.778 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 bash[61938]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-17T09:09:52.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:51.395264+0000 mgr.y (mgr.25024) 58 : cluster [DBG] pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:09:53.194 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 bash[61938]: Getting image source signatures 2024-09-17T09:09:53.228 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 bash[61938]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-17T09:09:53.229 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 bash[61938]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-17T09:09:53.229 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:52 smithi105 bash[61938]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-17T09:09:54.112 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:53 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:53.395684+0000 mgr.y (mgr.25024) 59 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:09:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:53.395684+0000 mgr.y (mgr.25024) 59 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:09:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:53.395684+0000 mgr.y (mgr.25024) 59 : cluster [DBG] pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:09:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:55 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:54.962323+0000 mon.b (mon.2) 169 : audit [DBG] from='client.? 172.21.15.105:0/2414525357' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:55.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:54 smithi105 bash[61938]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-17T09:09:55.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:55 smithi105 bash[61938]: Writing manifest to image destination 2024-09-17T09:09:55.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:54.962323+0000 mon.b (mon.2) 169 : audit [DBG] from='client.? 172.21.15.105:0/2414525357' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:55.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:54.962323+0000 mon.b (mon.2) 169 : audit [DBG] from='client.? 172.21.15.105:0/2414525357' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:09:55.699 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:55 smithi105 podman[61938]: 2024-09-17 09:09:55.433133132 +0000 UTC m=+3.044729633 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-17T09:09:56.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 podman[61938]: 2024-09-17 09:09:56.184310238 +0000 UTC m=+3.795906724 container create 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:55.307218+0000 mon.b (mon.2) 170 : audit [INF] from='client.? 172.21.15.105:0/305885822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:55.396570+0000 mgr.y (mgr.25024) 60 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:56.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:55.307218+0000 mon.b (mon.2) 170 : audit [INF] from='client.? 172.21.15.105:0/305885822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:56.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:55.396570+0000 mgr.y (mgr.25024) 60 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:56.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:55.307218+0000 mon.b (mon.2) 170 : audit [INF] from='client.? 172.21.15.105:0/305885822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:09:56.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:55.396570+0000 mgr.y (mgr.25024) 60 : cluster [DBG] pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 podman[61938]: 2024-09-17 09:09:56.408329914 +0000 UTC m=+4.019926405 container init 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 podman[61938]: 2024-09-17 09:09:56.411239945 +0000 UTC m=+4.022836434 container start 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.414Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.414Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-17T09:09:56.590 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=arp 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-17T09:09:56.591 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=edac 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-17T09:09:56.592 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=os 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-17T09:09:56.593 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=stat 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=time 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=uname 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.415Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.416Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b[62224]: ts=2024-09-17T09:09:56.416Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-17T09:09:56.594 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 bash[61938]: 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 2024-09-17T09:09:56.595 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:09:56 smithi105 systemd[1]: Started Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:57.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 3. 2024-09-17T09:09:57.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:57.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:09:57.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 podman[62376]: 2024-09-17 09:09:57.313111544 +0000 UTC m=+0.020518054 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:09:57.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 podman[62376]: 2024-09-17 09:09:57.441537942 +0000 UTC m=+0.148944422 container create aac038bab74fe7a625a89929c5f55fcc076106526931039951c97e51c6c44f17 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:58.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:56.546476+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:56.552526+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:09:56.554055+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:58.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:56.546476+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:56.552526+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:09:56.554055+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:58.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:56.546476+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:56.552526+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:09:58.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:09:56.554055+0000 mon.c (mon.1) 118 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:09:58.029 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 podman[62376]: 2024-09-17 09:09:57.668575372 +0000 UTC m=+0.375981858 container init aac038bab74fe7a625a89929c5f55fcc076106526931039951c97e51c6c44f17 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 podman[62376]: 2024-09-17 09:09:57.671320878 +0000 UTC m=+0.378727372 container start aac038bab74fe7a625a89929c5f55fcc076106526931039951c97e51c6c44f17 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.700Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.701Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.701Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.703Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:09:58.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.704Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.711Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.712Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.713Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.713Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=20 2024-09-17T09:09:58.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.714Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=17 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=18 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=19 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=20 maxSegment=20 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.715Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=12.033104ms 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:09:58.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=info ts=2024-09-17T09:09:57.717Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[62405]: level=error ts=2024-09-17T09:09:57.718Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 bash[62376]: aac038bab74fe7a625a89929c5f55fcc076106526931039951c97e51c6c44f17 2024-09-17T09:09:58.033 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:57 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:09:58.960 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:09:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:57.397041+0000 mgr.y (mgr.25024) 61 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:58.960 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:58 smithi105 podman[62425]: 2024-09-17 09:09:58.54033331 +0000 UTC m=+0.815434946 container remove aac038bab74fe7a625a89929c5f55fcc076106526931039951c97e51c6c44f17 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:09:58.960 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:58 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:09:58.960 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:09:58 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:09:59.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:09:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:57.397041+0000 mgr.y (mgr.25024) 61 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:09:59.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:09:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:57.397041+0000 mgr.y (mgr.25024) 61 : cluster [DBG] pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:09:59.397350+0000 mgr.y (mgr.25024) 62 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:00.000098+0000 mon.a (mon.0) 888 : cluster [WRN] Health detail: HEALTH_WARN 2 failed cephadm daemon(s) 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:00.000133+0000 mon.a (mon.0) 889 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 2 failed cephadm daemon(s) 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:00.000163+0000 mon.a (mon.0) 890 : cluster [WRN] daemon prometheus.a on smithi105 is in unknown state 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:00.000177+0000 mon.a (mon.0) 891 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:10:00.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:09:59.397350+0000 mgr.y (mgr.25024) 62 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:00.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:00.000098+0000 mon.a (mon.0) 888 : cluster [WRN] Health detail: HEALTH_WARN 2 failed cephadm daemon(s) 2024-09-17T09:10:00.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:00.000133+0000 mon.a (mon.0) 889 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 2 failed cephadm daemon(s) 2024-09-17T09:10:00.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:00.000163+0000 mon.a (mon.0) 890 : cluster [WRN] daemon prometheus.a on smithi105 is in unknown state 2024-09-17T09:10:00.612 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:00.000177+0000 mon.a (mon.0) 891 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:10:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:09:59.397350+0000 mgr.y (mgr.25024) 62 : cluster [DBG] pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:00.000098+0000 mon.a (mon.0) 888 : cluster [WRN] Health detail: HEALTH_WARN 2 failed cephadm daemon(s) 2024-09-17T09:10:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:00.000133+0000 mon.a (mon.0) 889 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 2 failed cephadm daemon(s) 2024-09-17T09:10:01.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:00.000163+0000 mon.a (mon.0) 890 : cluster [WRN] daemon prometheus.a on smithi105 is in unknown state 2024-09-17T09:10:01.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:00.000177+0000 mon.a (mon.0) 891 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:10:02.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:00.970719+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.121 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:00.983498+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:01.398246+0000 mgr.y (mgr.25024) 63 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:01.499974+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:01.505010+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:00.970719+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:00.983498+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:01.398246+0000 mgr.y (mgr.25024) 63 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:01.499974+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:01.505010+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:00.970719+0000 mon.a (mon.0) 892 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:00.983498+0000 mon.a (mon.0) 893 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:01.398246+0000 mgr.y (mgr.25024) 63 : cluster [DBG] pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:01.499974+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:02.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:01 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:01.505010+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:03.012073+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:03.021864+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:03.398637+0000 mgr.y (mgr.25024) 64 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:04.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:03.519577+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:03.525590+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:03.012073+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:03.021864+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:03.398637+0000 mgr.y (mgr.25024) 64 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:04.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:03.519577+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:03.525590+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:03.012073+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:03.021864+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:03.398637+0000 mgr.y (mgr.25024) 64 : cluster [DBG] pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:03.519577+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:04.373 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:03.525590+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:04.673438+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.001 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:04.681079+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:04.673438+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:04.681079+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:04.673438+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:04.681079+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:05.399486+0000 mgr.y (mgr.25024) 65 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.638273+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.646169+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.647732+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.648652+0000 mon.c (mon.1) 120 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:07.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.655262+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:05.399486+0000 mgr.y (mgr.25024) 65 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.638273+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.646169+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.647732+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.648652+0000 mon.c (mon.1) 120 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:07.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.655262+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:05.399486+0000 mgr.y (mgr.25024) 65 : cluster [DBG] pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.638273+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.646169+0000 mon.a (mon.0) 903 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.647732+0000 mon.c (mon.1) 119 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.648652+0000 mon.c (mon.1) 120 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:07.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.655262+0000 mon.a (mon.0) 904 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:07.976 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:06.649453+0000 mgr.y (mgr.25024) 66 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:07.976 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:06.650007+0000 mgr.y (mgr.25024) 67 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 90 B/s rd, 0 op/s 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:06.650248+0000 mgr.y (mgr.25024) 68 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.774538+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.777057+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.778326+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.779643+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.780811+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.781711+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.782617+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.783747+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.785075+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:07.977 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:06.786209+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:06.649453+0000 mgr.y (mgr.25024) 66 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:06.650007+0000 mgr.y (mgr.25024) 67 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 90 B/s rd, 0 op/s 2024-09-17T09:10:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:06.650248+0000 mgr.y (mgr.25024) 68 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.774538+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.777057+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.778326+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.779643+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.780811+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.781711+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.782617+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.783747+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.785075+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:06.786209+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:06.649453+0000 mgr.y (mgr.25024) 66 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:06.650007+0000 mgr.y (mgr.25024) 67 : cluster [DBG] pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 90 B/s rd, 0 op/s 2024-09-17T09:10:08.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:06.650248+0000 mgr.y (mgr.25024) 68 : cluster [DBG] pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.774538+0000 mon.c (mon.1) 121 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.777057+0000 mon.c (mon.1) 122 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.778326+0000 mon.c (mon.1) 123 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.779643+0000 mon.c (mon.1) 124 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.780811+0000 mon.c (mon.1) 125 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.781711+0000 mon.c (mon.1) 126 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.782617+0000 mon.c (mon.1) 127 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.783747+0000 mon.c (mon.1) 128 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.785075+0000 mon.c (mon.1) 129 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:06.786209+0000 mon.c (mon.1) 130 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:08.816 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:08 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Scheduled restart job, restart counter is at 4. 2024-09-17T09:10:08.816 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:08 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:08.816 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:08 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:09.196 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:07.998613+0000 mgr.y (mgr.25024) 69 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-17T09:10:09.196 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:08.244425+0000 mgr.y (mgr.25024) 70 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T09:10:09.196 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:08.650794+0000 mgr.y (mgr.25024) 71 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:07.998613+0000 mgr.y (mgr.25024) 69 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:08.244425+0000 mgr.y (mgr.25024) 70 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:08.650794+0000 mgr.y (mgr.25024) 71 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:07.998613+0000 mgr.y (mgr.25024) 69 : cephadm [INF] Upgrade: Updating prometheus.a 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:08.244425+0000 mgr.y (mgr.25024) 70 : cephadm [INF] Deploying daemon prometheus.a on smithi105 2024-09-17T09:10:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:08.650794+0000 mgr.y (mgr.25024) 71 : cluster [DBG] pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:09.485 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 podman[63741]: 2024-09-17 09:10:09.100453439 +0000 UTC m=+0.024142974 image pull de242295e2257c37c8cadfd962369228f8f10b2d48a44259b65fef44ad4f6490 docker.io/prom/prometheus:v2.18.1 2024-09-17T09:10:09.485 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 podman[63741]: 2024-09-17 09:10:09.237547207 +0000 UTC m=+0.161236750 container create c29585487514593cee721ee961f110a3169b42c676845cbb43c04151c1662337 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:09.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 podman[63741]: 2024-09-17 09:10:09.485320681 +0000 UTC m=+0.409010216 container init c29585487514593cee721ee961f110a3169b42c676845cbb43c04151c1662337 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 podman[63741]: 2024-09-17 09:10:09.488280363 +0000 UTC m=+0.411969897 container start c29585487514593cee721ee961f110a3169b42c676845cbb43c04151c1662337 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:337 msg="Starting Prometheus" version="(version=2.18.1, branch=HEAD, revision=ecee9c8abfd118f139014cb1b174b08db3f342cf)" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:338 build_context="(go=go1.14.2, user=root@2117a9e64a7e, date=20200507-16:51:47)" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:339 host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:340 fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.531Z caller=main.go:341 vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.533Z caller=main.go:678 msg="Starting TSDB ..." 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.533Z caller=web.go:523 component=web msg="Start listening for connections" address=:9095 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.535Z caller=head.go:575 component=tsdb msg="Replaying WAL, this may take awhile" 2024-09-17T09:10:09.779 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.536Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=1 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=2 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=3 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=4 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=5 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=6 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.545Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=7 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.546Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=8 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.546Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=9 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.546Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=10 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.546Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=11 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.547Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=12 maxSegment=21 2024-09-17T09:10:09.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.548Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=13 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.548Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=14 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.548Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=15 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=16 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=17 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=18 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=19 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=20 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.549Z caller=head.go:624 component=tsdb msg="WAL segment loaded" segment=21 maxSegment=21 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.550Z caller=head.go:627 component=tsdb msg="WAL replay completed" duration=14.478237ms 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.551Z caller=main.go:694 fs_type=XFS_SUPER_MAGIC 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.551Z caller=main.go:695 msg="TSDB started" 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.551Z caller=main.go:799 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:10:09.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:547 msg="Stopping scrape discovery manager..." 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:561 msg="Stopping notify discovery manager..." 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:583 msg="Stopping scrape manager..." 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:557 msg="Notify discovery manager stopped" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=manager.go:882 component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=manager.go:892 component="rule manager" msg="Rule manager stopped" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:543 msg="Scrape discovery manager stopped" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:577 msg="Scrape manager stopped" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=notifier.go:601 component=notifier msg="Stopping notification manager..." 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=info ts=2024-09-17T09:10:09.552Z caller=main.go:749 msg="Notifier manager stopped" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a[63832]: level=error ts=2024-09-17T09:10:09.552Z caller=main.go:758 err="error loading config from \"/etc/prometheus/prometheus.yml\": couldn't load configuration (--config.file=\"/etc/prometheus/prometheus.yml\"): parsing YAML file /etc/prometheus/prometheus.yml: yaml: unmarshal errors:\n line 14: field http_sd_configs not found in type config.plain\n line 27: field http_sd_configs not found in type config.plain\n line 31: field http_sd_configs not found in type config.plain\n line 45: field http_sd_configs not found in type config.plain\n line 49: field http_sd_configs not found in type config.plain" 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 bash[63741]: c29585487514593cee721ee961f110a3169b42c676845cbb43c04151c1662337 2024-09-17T09:10:09.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:09 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:10.198 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:09.567009+0000 mon.b (mon.2) 171 : audit [DBG] from='client.? 172.21.15.105:0/3716412550' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:09.567009+0000 mon.b (mon.2) 171 : audit [DBG] from='client.? 172.21.15.105:0/3716412550' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:09.567009+0000 mon.b (mon.2) 171 : audit [DBG] from='client.? 172.21.15.105:0/3716412550' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:10.500 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:10 smithi105 podman[63849]: 2024-09-17 09:10:10.197785146 +0000 UTC m=+0.633397938 container remove c29585487514593cee721ee961f110a3169b42c676845cbb43c04151c1662337 (image=docker.io/prom/prometheus:v2.18.1, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:10.500 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:10 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Main process exited, code=exited, status=1/FAILURE 2024-09-17T09:10:10.501 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:10 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service: Failed with result 'exit-code'. 2024-09-17T09:10:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:09.899382+0000 mon.b (mon.2) 172 : audit [INF] from='client.? 172.21.15.105:0/1844911246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:10.651664+0000 mgr.y (mgr.25024) 72 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:09.899382+0000 mon.b (mon.2) 172 : audit [INF] from='client.? 172.21.15.105:0/1844911246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:10.651664+0000 mgr.y (mgr.25024) 72 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:09.899382+0000 mon.b (mon.2) 172 : audit [INF] from='client.? 172.21.15.105:0/1844911246' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:10.651664+0000 mgr.y (mgr.25024) 72 : cluster [DBG] pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:12.652210+0000 mgr.y (mgr.25024) 73 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:12.652210+0000 mgr.y (mgr.25024) 73 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:12.652210+0000 mgr.y (mgr.25024) 73 : cluster [DBG] pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:15.000 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:14 smithi105 systemd[1]: Stopped Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:15.000 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:14 smithi105 systemd[1]: Starting Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 podman[64125]: 2024-09-17 09:10:14.903231806 +0000 UTC m=+0.016839964 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 podman[64125]: 2024-09-17 09:10:15.015241747 +0000 UTC m=+0.128849888 container create f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 podman[64125]: 2024-09-17 09:10:15.209014686 +0000 UTC m=+0.322622838 container init f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 podman[64125]: 2024-09-17 09:10:15.212305883 +0000 UTC m=+0.325914024 container start f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.237Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.237Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-17T09:10:15.251 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.237Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.237Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.237Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.239Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.239Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.239Z caller=dir_locker.go:77 level=warn component=tsdb msg="A lockfile from a previous execution already existed. It was replaced" file=/prometheus/lock 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.240Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.240Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.241Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.241Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.604µs 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.241Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.242Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=22 2024-09-17T09:10:15.252 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.248Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=22 2024-09-17T09:10:15.253 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.249Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=22 2024-09-17T09:10:15.253 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.249Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=22 2024-09-17T09:10:15.253 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.249Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=22 2024-09-17T09:10:15.253 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.249Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=22 2024-09-17T09:10:15.253 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.250Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=22 2024-09-17T09:10:15.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.251Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=7 maxSegment=22 2024-09-17T09:10:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.252Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=8 maxSegment=22 2024-09-17T09:10:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.252Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=9 maxSegment=22 2024-09-17T09:10:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.252Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=10 maxSegment=22 2024-09-17T09:10:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.252Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=11 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=12 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=13 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=14 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=15 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=16 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.253Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=17 maxSegment=22 2024-09-17T09:10:15.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.254Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=18 maxSegment=22 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.255Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=19 maxSegment=22 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.255Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=20 maxSegment=22 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.256Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=21 maxSegment=22 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.256Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=22 maxSegment=22 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.256Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.29µs wal_replay_duration=14.390431ms wbl_replay_duration=99ns total_replay_duration=14.443155ms 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.257Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.257Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.257Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-17T09:10:15.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.275Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.91431ms db_storage=787ns remote_storage=1.447µs web_handler=463ns query_engine=550ns scrape=370.808µs scrape_sd=85.198µs notify=13.448µs notify_sd=15.52µs rules=17.132974ms tracing=5.433µs 2024-09-17T09:10:15.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.275Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-17T09:10:15.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:10:15.275Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-17T09:10:15.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 bash[64125]: f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c 2024-09-17T09:10:15.532 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:10:15 smithi105 systemd[1]: Started Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:15.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:14.653089+0000 mgr.y (mgr.25024) 74 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:15.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:15.360478+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:15.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:15.367583+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:15.974 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:15.369157+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:14.653089+0000 mgr.y (mgr.25024) 74 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:15.360478+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:15.367583+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:15.369157+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:14.653089+0000 mgr.y (mgr.25024) 74 : cluster [DBG] pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 110 B/s rd, 0 op/s 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:15.360478+0000 mon.a (mon.0) 905 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:16.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:15.367583+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:16.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:15.369157+0000 mon.c (mon.1) 131 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:16.653625+0000 mgr.y (mgr.25024) 75 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:10:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:16.653625+0000 mgr.y (mgr.25024) 75 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:10:18.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:16.653625+0000 mgr.y (mgr.25024) 75 : cluster [DBG] pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:10:18.258 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (13m) 17s ago 21m 19.2M - 0.20.0 0881eb8f169f b32a9db2c611 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (13m) 15s ago 21m 35.8M - 6.7.4 557c83e11646 3f85a6d8d4f5 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 15s ago 20m - - 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (46s) 15s ago 25m 435M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (10m) 17s ago 26m 475M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (26m) 17s ago 26m 88.1M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (25m) 15s ago 25m 67.1M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:10:18.909 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (25m) 17s ago 25m 59.0M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (30s) 17s ago 22m 3590k - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (23s) 15s ago 21m 5486k - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (25m) 17s ago 25m 49.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (24m) 17s ago 24m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (24m) 17s ago 24m 46.1M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (24m) 17s ago 24m 50.1M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (23m) 15s ago 23m 53.7M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (23m) 15s ago 23m 47.1M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (23m) 15s ago 23m 48.5M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (22m) 15s ago 22m 52.5M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 starting - - - - 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (20m) 17s ago 20m 41.4M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:10:18.910 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (21m) 15s ago 21m 42.8M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:18.244721+0000 mgr.y (mgr.25024) 76 : audit [DBG] from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:18.577546+0000 mgr.y (mgr.25024) 77 : audit [DBG] from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:18.653995+0000 mgr.y (mgr.25024) 78 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:18.244721+0000 mgr.y (mgr.25024) 76 : audit [DBG] from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:18.577546+0000 mgr.y (mgr.25024) 77 : audit [DBG] from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:18.653995+0000 mgr.y (mgr.25024) 78 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:18.244721+0000 mgr.y (mgr.25024) 76 : audit [DBG] from='client.25330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:18.577546+0000 mgr.y (mgr.25024) 77 : audit [DBG] from='client.25336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:18.653995+0000 mgr.y (mgr.25024) 78 : cluster [DBG] pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:19.303 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:10:19.304 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:10:19.638 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T09:10:19.639 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading prometheus daemons", 2024-09-17T09:10:19.639 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:10:19.639 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:18.901365+0000 mgr.y (mgr.25024) 79 : audit [DBG] from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:19.302824+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.27:0/1866838185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:19.637704+0000 mgr.y (mgr.25024) 80 : audit [DBG] from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:18.901365+0000 mgr.y (mgr.25024) 79 : audit [DBG] from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:19.302824+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.27:0/1866838185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:20.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:19.637704+0000 mgr.y (mgr.25024) 80 : audit [DBG] from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:18.901365+0000 mgr.y (mgr.25024) 79 : audit [DBG] from='client.25342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:19.302824+0000 mon.a (mon.0) 907 : audit [DBG] from='client.? 172.21.15.27:0/1866838185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:19.637704+0000 mgr.y (mgr.25024) 80 : audit [DBG] from='client.15414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:21.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:20.436390+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:20.442691+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:20.654743+0000 mgr.y (mgr.25024) 81 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:21.579 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:20.943940+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.580 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:20.950611+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:20.436390+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:20.442691+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:20.654743+0000 mgr.y (mgr.25024) 81 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:20.943940+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:20.950611+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:20.436390+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:20.442691+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:20.654743+0000 mgr.y (mgr.25024) 81 : cluster [DBG] pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:21.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:20.943940+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:21.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:20.950611+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:23.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:22.655091+0000 mgr.y (mgr.25024) 82 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:24.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:22.655091+0000 mgr.y (mgr.25024) 82 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:22.655091+0000 mgr.y (mgr.25024) 82 : cluster [DBG] pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:24.420801+0000 mon.b (mon.2) 173 : audit [DBG] from='client.? 172.21.15.105:0/1755306600' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:24.420801+0000 mon.b (mon.2) 173 : audit [DBG] from='client.? 172.21.15.105:0/1755306600' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:25.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:24.420801+0000 mon.b (mon.2) 173 : audit [DBG] from='client.? 172.21.15.105:0/1755306600' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:24.655967+0000 mgr.y (mgr.25024) 83 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:24.758852+0000 mon.b (mon.2) 174 : audit [INF] from='client.? 172.21.15.105:0/2488085094' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.047994+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.055245+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.057286+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.058290+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.064663+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.147730+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.186821+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:25.832 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.189652+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.191073+0000 mon.c (mon.1) 137 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.192427+0000 mon.c (mon.1) 138 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.193619+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.194512+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.195473+0000 mon.c (mon.1) 141 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.196688+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.198096+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.199293+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.833 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.200403+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:24.655967+0000 mgr.y (mgr.25024) 83 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:24.758852+0000 mon.b (mon.2) 174 : audit [INF] from='client.? 172.21.15.105:0/2488085094' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.047994+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.055245+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.057286+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.058290+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.064663+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.147730+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.186821+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.189652+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.191073+0000 mon.c (mon.1) 137 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.834 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.192427+0000 mon.c (mon.1) 138 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.193619+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.194512+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.195473+0000 mon.c (mon.1) 141 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.196688+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.198096+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.199293+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.835 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.200403+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:24.655967+0000 mgr.y (mgr.25024) 83 : cluster [DBG] pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:24.758852+0000 mon.b (mon.2) 174 : audit [INF] from='client.? 172.21.15.105:0/2488085094' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.047994+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.055245+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.057286+0000 mon.c (mon.1) 132 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.058290+0000 mon.c (mon.1) 133 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.064663+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.147730+0000 mon.c (mon.1) 134 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.186821+0000 mon.c (mon.1) 135 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:25.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.189652+0000 mon.c (mon.1) 136 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.191073+0000 mon.c (mon.1) 137 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.192427+0000 mon.c (mon.1) 138 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.193619+0000 mon.c (mon.1) 139 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.194512+0000 mon.c (mon.1) 140 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.195473+0000 mon.c (mon.1) 141 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.196688+0000 mon.c (mon.1) 142 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.198096+0000 mon.c (mon.1) 143 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.199293+0000 mon.c (mon.1) 144 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:25.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.200403+0000 mon.c (mon.1) 145 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:25.059098+0000 mgr.y (mgr.25024) 84 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:25.059722+0000 mgr.y (mgr.25024) 85 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:25.059984+0000 mgr.y (mgr.25024) 86 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:25.148203+0000 mgr.y (mgr.25024) 87 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:26.053083+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:25.059098+0000 mgr.y (mgr.25024) 84 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:25.059722+0000 mgr.y (mgr.25024) 85 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:10:27.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:25.059984+0000 mgr.y (mgr.25024) 86 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:27.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:25.148203+0000 mgr.y (mgr.25024) 87 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:27.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:26.053083+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:10:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:25.059098+0000 mgr.y (mgr.25024) 84 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:25.059722+0000 mgr.y (mgr.25024) 85 : cluster [DBG] pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:10:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:25.059984+0000 mgr.y (mgr.25024) 86 : cluster [DBG] pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:25.148203+0000 mgr.y (mgr.25024) 87 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:10:27.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:26.053083+0000 mon.a (mon.0) 915 : cluster [WRN] Health check update: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:10:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:26.488210+0000 mgr.y (mgr.25024) 88 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-17T09:10:27.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:26.495655+0000 mgr.y (mgr.25024) 89 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T09:10:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:26.488210+0000 mgr.y (mgr.25024) 88 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-17T09:10:27.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:26.495655+0000 mgr.y (mgr.25024) 89 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T09:10:28.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:26.488210+0000 mgr.y (mgr.25024) 88 : cephadm [INF] Upgrade: Updating alertmanager.a 2024-09-17T09:10:28.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:26.495655+0000 mgr.y (mgr.25024) 89 : cephadm [INF] Deploying daemon alertmanager.a on smithi027 2024-09-17T09:10:29.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:27.060378+0000 mgr.y (mgr.25024) 90 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:27.060378+0000 mgr.y (mgr.25024) 90 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:29.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:27.060378+0000 mgr.y (mgr.25024) 90 : cluster [DBG] pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:29.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:29 smithi027 systemd[1]: Stopping Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:29.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a[50142]: level=info ts=2024-09-17T09:10:29.591Z caller=main.go:536 msg="Received SIGTERM, exiting gracefully..." 2024-09-17T09:10:29.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:29 smithi027 podman[62478]: 2024-09-17 09:10:29.612225245 +0000 UTC m=+0.138587967 container died b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:30.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:29 smithi027 podman[62478]: 2024-09-17 09:10:29.982680907 +0000 UTC m=+0.509043618 container cleanup b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:30.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:29 smithi027 bash[62478]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a 2024-09-17T09:10:30.610 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:30 smithi027 podman[62491]: 2024-09-17 09:10:30.258703285 +0000 UTC m=+0.645896906 container remove b32a9db2c61177185cf2e68fbea8cb8810ed620eb1e6d4aab7b140fb0c1fa8bc (image=docker.io/prom/alertmanager:v0.20.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager.a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:30.610 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:30 smithi027 podman[62491]: 2024-09-17 09:10:30.334381946 +0000 UTC m=+0.721575585 volume remove b47ee2087e1864d9189c6a3493c09ce200a1f921f78c6baf262ddec48a386354 2024-09-17T09:10:31.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:29.060830+0000 mgr.y (mgr.25024) 91 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:31.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:29.060830+0000 mgr.y (mgr.25024) 91 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:31.009 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:30 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T09:10:31.010 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:30 smithi027 systemd[1]: Stopped Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:31.010 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:30 smithi027 systemd[1]: Starting Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:29.060830+0000 mgr.y (mgr.25024) 91 : cluster [DBG] pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:31.328 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 podman[62631]: 2024-09-17 09:10:30.947910337 +0000 UTC m=+0.016579636 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-17T09:10:31.328 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 podman[62631]: 2024-09-17 09:10:31.052473011 +0000 UTC m=+0.121142300 volume create 2df0deeaf3fa6e98f759af367c3ac6b91661d7102bde99700a8f46aca7f0ada8 2024-09-17T09:10:31.328 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 podman[62631]: 2024-09-17 09:10:31.153007982 +0000 UTC m=+0.221677269 container create 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 podman[62631]: 2024-09-17 09:10:31.455757329 +0000 UTC m=+0.524426624 container init 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 podman[62631]: 2024-09-17 09:10:31.458880316 +0000 UTC m=+0.527549616 container start 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.472Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.472Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.473Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.27 port=9094 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.475Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.504Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T09:10:31.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.504Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-17T09:10:31.759 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.507Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-17T09:10:31.759 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:31.507Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-17T09:10:31.759 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 bash[62631]: 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 2024-09-17T09:10:31.759 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:31 smithi027 systemd[1]: Started Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:32.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:31.061454+0000 mgr.y (mgr.25024) 92 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:32.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:31.616273+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:32.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:31.622085+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:32.942 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:31.623532+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:31.061454+0000 mgr.y (mgr.25024) 92 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:31.616273+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:31.622085+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:31.623532+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:31.061454+0000 mgr.y (mgr.25024) 92 : cluster [DBG] pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:31.616273+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:31.622085+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:33.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:31.623532+0000 mon.c (mon.1) 146 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:33.757 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:33.475Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000367443s 2024-09-17T09:10:34.235 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:33.061845+0000 mgr.y (mgr.25024) 93 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:34.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:33.061845+0000 mgr.y (mgr.25024) 93 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:33.061845+0000 mgr.y (mgr.25024) 93 : cluster [DBG] pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:36.418 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:35.062539+0000 mgr.y (mgr.25024) 94 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:36.418 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:35.804116+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:36.418 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:35.809269+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:35.062539+0000 mgr.y (mgr.25024) 94 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:35.804116+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:35.809269+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:35.062539+0000 mgr.y (mgr.25024) 94 : cluster [DBG] pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:35.804116+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:36.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:35.809269+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.575 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:36.326354+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.575 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:36.331335+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:36.326354+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:36.331335+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.710 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:36.326354+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:37.710 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:36.331335+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:38.491 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:37.062848+0000 mgr.y (mgr.25024) 95 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:38.491 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:37.771298+0000 mon.b (mon.2) 175 : audit [DBG] from='client.? 172.21.15.105:0/4217261822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:38.491 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:38.091144+0000 mon.b (mon.2) 176 : audit [INF] from='client.? 172.21.15.105:0/3129364040' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:37.062848+0000 mgr.y (mgr.25024) 95 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:37.771298+0000 mon.b (mon.2) 175 : audit [DBG] from='client.? 172.21.15.105:0/4217261822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:38.091144+0000 mon.b (mon.2) 176 : audit [INF] from='client.? 172.21.15.105:0/3129364040' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:37.062848+0000 mgr.y (mgr.25024) 95 : cluster [DBG] pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:37.771298+0000 mon.b (mon.2) 175 : audit [DBG] from='client.? 172.21.15.105:0/4217261822' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:38.091144+0000 mon.b (mon.2) 176 : audit [INF] from='client.? 172.21.15.105:0/3129364040' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:39.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:38.381025+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:39.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:38.387738+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:39.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:38.381025+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:39.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:38.387738+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:39.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:38.381025+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:39.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:38.387738+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:39.063148+0000 mgr.y (mgr.25024) 96 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.595048+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.605006+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.608391+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.609152+0000 mon.c (mon.1) 148 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:39.609799+0000 mgr.y (mgr.25024) 97 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:39.610595+0000 mgr.y (mgr.25024) 98 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 97 B/s rd, 0 op/s 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:39.610804+0000 mgr.y (mgr.25024) 99 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.617190+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.700005+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:40.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.700400+0000 mgr.y (mgr.25024) 100 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.738851+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.741438+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.742784+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.744063+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.745243+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.746108+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.746983+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.748096+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.749381+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.750464+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.751528+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:39.752574+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:39.063148+0000 mgr.y (mgr.25024) 96 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.595048+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.605006+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.608391+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.609152+0000 mon.c (mon.1) 148 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:39.609799+0000 mgr.y (mgr.25024) 97 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:39.610595+0000 mgr.y (mgr.25024) 98 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 97 B/s rd, 0 op/s 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:39.610804+0000 mgr.y (mgr.25024) 99 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.617190+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.700005+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.700400+0000 mgr.y (mgr.25024) 100 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.738851+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.741438+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.742784+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.744063+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.745243+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:39.063148+0000 mgr.y (mgr.25024) 96 : cluster [DBG] pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.595048+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.605006+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.608391+0000 mon.c (mon.1) 147 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.609152+0000 mon.c (mon.1) 148 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:39.609799+0000 mgr.y (mgr.25024) 97 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:39.610595+0000 mgr.y (mgr.25024) 98 : cluster [DBG] pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 97 B/s rd, 0 op/s 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:39.610804+0000 mgr.y (mgr.25024) 99 : cluster [DBG] pgmap v53: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.617190+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.700005+0000 mon.c (mon.1) 149 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:41.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.700400+0000 mgr.y (mgr.25024) 100 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.738851+0000 mon.c (mon.1) 150 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.741438+0000 mon.c (mon.1) 151 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.742784+0000 mon.c (mon.1) 152 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.744063+0000 mon.c (mon.1) 153 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.745243+0000 mon.c (mon.1) 154 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.746108+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.746983+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.748096+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.749381+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.750464+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.751528+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:39.752574+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.746108+0000 mon.c (mon.1) 155 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.746983+0000 mon.c (mon.1) 156 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.748096+0000 mon.c (mon.1) 157 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.749381+0000 mon.c (mon.1) 158 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.750464+0000 mon.c (mon.1) 159 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.751528+0000 mon.c (mon.1) 160 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:39.752574+0000 mon.c (mon.1) 161 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:41.758 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:10:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:10:41.478Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003156265s 2024-09-17T09:10:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:41.055445+0000 mgr.y (mgr.25024) 101 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-17T09:10:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:41.101264+0000 mgr.y (mgr.25024) 102 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T09:10:42.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:41.611314+0000 mgr.y (mgr.25024) 103 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:41.055445+0000 mgr.y (mgr.25024) 101 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:41.101264+0000 mgr.y (mgr.25024) 102 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:41.611314+0000 mgr.y (mgr.25024) 103 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:41.055445+0000 mgr.y (mgr.25024) 101 : cephadm [INF] Upgrade: Updating grafana.a 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:41.101264+0000 mgr.y (mgr.25024) 102 : cephadm [INF] Deploying daemon grafana.a on smithi105 2024-09-17T09:10:43.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:41.611314+0000 mgr.y (mgr.25024) 103 : cluster [DBG] pgmap v54: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:43.611777+0000 mgr.y (mgr.25024) 104 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:43.611777+0000 mgr.y (mgr.25024) 104 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:44.263 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:43.611777+0000 mgr.y (mgr.25024) 104 : cluster [DBG] pgmap v55: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:44.263 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 systemd[1]: Stopping Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:44.264 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-09-17T09:10:44.264 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped Stream Manager" 2024-09-17T09:10:44.264 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped BackendPluginManager" logger=server reason="context canceled" 2024-09-17T09:10:44.264 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped CleanUpService" logger=server reason="context canceled" 2024-09-17T09:10:44.264 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped PluginManager" logger=server reason="context canceled" 2024-09-17T09:10:44.265 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped InternalMetricsService" logger=server reason="context canceled" 2024-09-17T09:10:44.265 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped NotificationService" logger=server reason="context canceled" 2024-09-17T09:10:44.265 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped provisioningServiceImpl" logger=server reason="context canceled" 2024-09-17T09:10:44.265 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped RemoteCache" logger=server reason="context canceled" 2024-09-17T09:10:44.265 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped AlertEngine" logger=server reason="context canceled" 2024-09-17T09:10:44.266 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped UsageStatsService" logger=server reason="context canceled" 2024-09-17T09:10:44.266 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a[44086]: t=2024-09-17T09:10:43+0000 lvl=info msg="Stopped UserAuthTokenService" logger=server reason="context canceled" 2024-09-17T09:10:44.266 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:43 smithi105 podman[66801]: 2024-09-17 09:10:43.99711407 +0000 UTC m=+0.127390908 container died 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, org.label-schema.schema-version=1.0, io.buildah.version=1.19.8, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, maintainer=Paul Cuzner ) 2024-09-17T09:10:44.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:44 smithi105 podman[66801]: 2024-09-17 09:10:44.377954158 +0000 UTC m=+0.508230995 container cleanup 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, description=Ceph Grafana Container, maintainer=Paul Cuzner , org.label-schema.vendor=CentOS, io.buildah.version=1.19.8, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T09:10:44.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:44 smithi105 bash[66801]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a 2024-09-17T09:10:44.893 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:44 smithi105 podman[66813]: 2024-09-17 09:10:44.620740701 +0000 UTC m=+0.621961891 container remove 3f85a6d8d4f5cc2e5834aaa36027097ba0e499f500a6d6b8b45ecd8d13138afc (image=docker.io/ceph/ceph-grafana:6.7.4, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana.a, io.buildah.version=1.19.8, description=Ceph Grafana Container, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Paul Cuzner , org.label-schema.license=GPLv2, summary=Grafana Container configured for Ceph mgr/dashboard integration, org.label-schema.name=CentOS Base Image) 2024-09-17T09:10:45.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:44 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-17T09:10:45.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:44 smithi105 systemd[1]: Stopped Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:45.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 systemd[1]: Starting Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:10:45.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 podman[66956]: 2024-09-17 09:10:45.223290294 +0000 UTC m=+0.015191836 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-17T09:10:45.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 podman[66956]: 2024-09-17 09:10:45.330424725 +0000 UTC m=+0.122326253 container create 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, vcs-type=git, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vendor=Red Hat, Inc., maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T09:10:45.574 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 podman[66956]: 2024-09-17 09:10:45.574257422 +0000 UTC m=+0.366158954 container init 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, name=ubi8, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, release=239.1651231664, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner ) 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 podman[66956]: 2024-09-17 09:10:45.576990178 +0000 UTC m=+0.368891712 container start 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, name=ubi8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.buildah.version=1.32.0, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, version=8.5, build-date=2022-04-29T11:48:11.055897) 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 bash[66956]: 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 systemd[1]: Started Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731005929Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731234039Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731257544Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731272647Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731283339Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731293868Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731303646Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-17T09:10:45.826 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731313735Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-17T09:10:45.827 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731323955Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-17T09:10:45.827 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731333776Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-17T09:10:45.827 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731343292Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-17T09:10:45.827 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731352567Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-17T09:10:45.827 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731362172Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=settings t=2024-09-17T09:10:45.731371795Z level=info msg="App mode production" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore t=2024-09-17T09:10:45.731426557Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore t=2024-09-17T09:10:45.731449368Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.73229986Z level=info msg="Starting DB migrations" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.732393897Z level=info msg="Executing migration" id="create migration_log table" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.732927201Z level=info msg="Executing migration" id="create user table" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.733319787Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.734187724Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.735026887Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.735861327Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.736690551Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.7375612Z level=info msg="Executing migration" id="create user table v2" 2024-09-17T09:10:45.828 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.738391595Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.739261514Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.740111351Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.740930252Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.74179977Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.742692173Z level=info msg="Executing migration" id="Update user table charset" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.743090124Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.74392154Z level=info msg="Executing migration" id="Add missing user data" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.744799282Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.745644011Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.746065319Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.746900728Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.748012129Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-17T09:10:45.829 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.748895794Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.749308028Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.750146751Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.751032868Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.751844027Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.752709558Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.753571016Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.755835991Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.758676529Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.76135323Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.764047294Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.766912129Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.769506405Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-17T09:10:45.830 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.772213761Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.774921329Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.777646845Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.780338925Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.782943419Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.783308337Z level=info msg="Executing migration" id="create star table" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.785974121Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.78642592Z level=info msg="Executing migration" id="create org table v1" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.788597288Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.791205804Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.793839394Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.79654311Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-17T09:10:45.831 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.796987073Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.79911199Z level=info msg="Executing migration" id="Update org table charset" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.801778088Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.804378334Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.807000729Z level=info msg="Executing migration" id="create dashboard table" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.807402082Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.808182739Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-17T09:10:45.832 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.809002625Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.809784689Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.810206468Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.810985184Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.811983861Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.812370839Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.813195701Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.813622553Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.814384319Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.814781852Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.815209789Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.815968614Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.81644332Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.816892952Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-17T09:10:45.833 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.81731091Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.817761516Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.818181233Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.818624378Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.819372077Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.819703618Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.820215711Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.820698509Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.82154055Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.822383239Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.822724821Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.823509246Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.82438055Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.825159217Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-17T09:10:45.834 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.825642435Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-17T09:10:45.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.826496433Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-17T09:10:45.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.827441462Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-17T09:10:45.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.828318497Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-17T09:10:45.835 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.828757851Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.829615954Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.832314579Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.835038511Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.835575458Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.838287882Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.841093614Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.843725957Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-17T09:10:46.076 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.844230395Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.847046117Z level=info msg="Executing migration" id="create data_source table" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.849926259Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.850367754Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.852892542Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.85564771Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.858396901Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.861089985Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.8615021Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.864286953Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-17T09:10:46.077 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.864702107Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-17T09:10:46.078 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.867089932Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-17T09:10:46.078 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.869836462Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.870354852Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.870622168Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.870894184Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.873095787Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.875810467Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.876248857Z level=info msg="Executing migration" id="Add uid column" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.876792009Z level=info msg="Executing migration" id="Update uid value" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.877115743Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.879528031Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-17T09:10:46.079 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.87997813Z level=info msg="Executing migration" id="create api_key table" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.882255344Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.884941618Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.885400889Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.887977869Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.890726426Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.891152377Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.891554124Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.892583327Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.892992643Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.893406796Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-17T09:10:46.080 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.893855218Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.894285377Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.894584959Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.89517765Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.895468087Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.896049153Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.896591506Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.896888966Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.897453637Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.898031081Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.89842369Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-17T09:10:46.081 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.898786035Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.899212858Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.899691656Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.900114817Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.900540353Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.900823286Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.90113673Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.901694165Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.902345775Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.902677982Z level=info msg="Executing migration" id="create quota table v1" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.903131441Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-17T09:10:46.082 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.903579261Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.90387347Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.904292038Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.904721039Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.90529644Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.905612892Z level=info msg="Executing migration" id="create session table" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.906066524Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.906367316Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.906653527Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.907068698Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.907462876Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.907737104Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-17T09:10:46.083 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.908089125Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.908467221Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.908783441Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.909225112Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.909512934Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.910145318Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.910464412Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.911079163Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.91168126Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.912005341Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.912490025Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.913091536Z level=info msg="Executing migration" id="create alert table v1" 2024-09-17T09:10:46.084 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.91570178Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.918412228Z level=info msg="Executing migration" id="add index alert state" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.921171299Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.923784312Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.924514527Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.927336066Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.930078401Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.932983653Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.935692736Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-17T09:10:46.085 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.9385116Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-17T09:10:46.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.941559955Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-17T09:10:46.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.944638877Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-17T09:10:46.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.947334572Z level=info msg="Executing migration" id="Add column is_default" 2024-09-17T09:10:46.086 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.949955902Z level=info msg="Executing migration" id="Add column frequency" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.952714009Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.955503077Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.95815796Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.960892558Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.96338313Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.966144795Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.968864733Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.971621092Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.974262656Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.977062723Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.979759037Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.982529605Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-17T09:10:46.087 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.98513329Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.985475208Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.988167993Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.990924291Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.993548711Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.9939652Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.994835916Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.995640766Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.996485429Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.996950112Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.997791035Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.99864136Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-17T09:10:46.088 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:45.999513124Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.00035435Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.000957116Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.001851849Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.002646195Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.003486738Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.004209696Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.005054441Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.005897097Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.006741719Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.008658186Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-17T09:10:46.089 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.009451926Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.010303222Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.011345818Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.011732887Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.012586036Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.013481889Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.014313399Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.01477399Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.015579732Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.016406749Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.017130029Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-17T09:10:46.090 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.017579622Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.017918299Z level=info msg="Executing migration" id="Move region to single row" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.018402517Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.018905085Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.019326788Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.019778183Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.020208531Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.020620547Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.021126855Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.021460653Z level=info msg="Executing migration" id="create test_data table" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.024163498Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-17T09:10:46.091 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.026953655Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.029784721Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.032512748Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.035318274Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.038078714Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.04087357Z level=info msg="Executing migration" id="create team table" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.041275156Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.044024822Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.04693186Z level=info msg="Executing migration" id="create team member table" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.049832732Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-17T09:10:46.092 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.050298902Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.052983736Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.055773412Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.058474842Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.059229227Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.062025054Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.064369748Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.067111305Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.06983368Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.072602674Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.075237875Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-17T09:10:46.093 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.077938202Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-17T09:10:46.094 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.08073584Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.083386834Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.086063607Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.088801704Z level=info msg="Executing migration" id="create tag table" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.091521384Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.09433604Z level=info msg="Executing migration" id="create login attempt table" 2024-09-17T09:10:46.327 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.096952537Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.099724308Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.10056702Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.102673545Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.103551967Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.104337937Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.105160312Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.106021709Z level=info msg="Executing migration" id="create user auth table" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.106932149Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.107764917Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.10858966Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.10947443Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-17T09:10:46.328 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.110321638Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-17T09:10:46.329 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.111175121Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.112047841Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.112895361Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.113753409Z level=info msg="Executing migration" id="create server_lock table" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.114607804Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.11543062Z level=info msg="Executing migration" id="create user auth token table" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.116300381Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.117185458Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-17T09:10:46.330 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.118028098Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.118887923Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.119770227Z level=info msg="Executing migration" id="create cache_data table" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.120637011Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.121480016Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.122390171Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.12321501Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.124049405Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.124908482Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.125773437Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.126659691Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.127507174Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.128403815Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-17T09:10:46.331 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.129202926Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.130047113Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.130911309Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.133855221Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.134964453Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.135406852Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.135680853Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.136099561Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.13651811Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.136910031Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.137255464Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.137641956Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.138082723Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-17T09:10:46.332 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.138561762Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.1391615Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.1403463Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.140961366Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.141552173Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.146868764Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.150458918Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.151082714Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.151648417Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-17T09:10:46.333 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.152876838Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.15349159Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.154348079Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.155263655Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.156076907Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.156911632Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.158144569Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.159302396Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.160538227Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.161432756Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.162254194Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.163576279Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-17T09:10:46.334 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.164780983Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.165647446Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.166933724Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.167927638Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.168791258Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.169667662Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.170524194Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.173160246Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.173564776Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.176441368Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.179293073Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.181957556Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.184779313Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-17T09:10:46.335 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.187540234Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.190309824Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.193144091Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.195968532Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.198968967Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.20184719Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.204530386Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.2072244Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.209978441Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.21278571Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.215628225Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.21836353Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-17T09:10:46.336 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.221067212Z level=info msg="Executing migration" id="create alert_image table" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.223799789Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.226679656Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.229285419Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.232088634Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.232606443Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.23270666Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.235457024Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.238014221Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.240681363Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.240838338Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.240963135Z level=info msg="alerts found to migrate" alerts=0 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.243449232Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.245909107Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-17T09:10:46.337 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.248652254Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.251467677Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.254118763Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.256723214Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.259373427Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.262295305Z level=info msg="Executing migration" id="create data_keys table" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.264875392Z level=info msg="Executing migration" id="create secrets table" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.268666025Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.272959095Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.273973814Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.274826209Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.27894614Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.283041378Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.28391719Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.284644877Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-17T09:10:46.338 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.285493372Z level=info msg="Executing migration" id="create permission table" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.2863753Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.287236234Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.288028992Z level=info msg="Executing migration" id="create role table" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.288842349Z level=info msg="Executing migration" id="add column display_name" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.289777025Z level=info msg="Executing migration" id="add column group_name" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.290710892Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.291559405Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.292404299Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.293228774Z level=info msg="Executing migration" id="create team role table" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.294034091Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.294926072Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.295918959Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-17T09:10:46.339 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.296724129Z level=info msg="Executing migration" id="create user role table" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.297686755Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.298432133Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.29928082Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.300254523Z level=info msg="Executing migration" id="create builtin role table" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.301400817Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.301850221Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-17T09:10:46.340 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.302723932Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.303673484Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.304152439Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.3049587Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.305370813Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.305760504Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.306134677Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.306556702Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.30751458Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.308335528Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.308758276Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-17T09:10:46.341 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.309053982Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.309410893Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.30982761Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.310187292Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.310457133Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.31078134Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.311178336Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.311597684Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.312566868Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.312874581Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.313283343Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.313717095Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.314133243Z level=info msg="Executing migration" id="add correlation config column" 2024-09-17T09:10:46.342 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.315128601Z level=info msg="Executing migration" id="create entity_events table" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.315541899Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.315951149Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.31603417Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.316302814Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.316375881Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.316611542Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.317003621Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.317503695Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.318486777Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.32117059Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.323708251Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.326509794Z level=info msg="Executing migration" id="Drop public config table" 2024-09-17T09:10:46.343 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.329092364Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-17T09:10:46.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.329569215Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-17T09:10:46.344 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.330408445Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-17T09:10:46.588 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.333408618Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-17T09:10:46.588 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.336218421Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-17T09:10:46.588 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.339264429Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-17T09:10:46.588 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.342033689Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.344931886Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.34792314Z level=info msg="Executing migration" id="add share column" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.35089096Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.353733643Z level=info msg="Executing migration" id="create file table" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.356491953Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.359335763Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.362187507Z level=info msg="Executing migration" id="create file_meta table" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.364943986Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.367839732Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.370835666Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.373605423Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-17T09:10:46.589 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.376494467Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.379114512Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.381903353Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.384676743Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.38735778Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.389912532Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.392426667Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.395201064Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.39789475Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.400673227Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.403512287Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.413421286Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.416184796Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.416654905Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-17T09:10:46.590 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.420059319Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.4227261Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.42526583Z level=info msg="Executing migration" id="create folder table" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.425873869Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.42844962Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.431125814Z level=info msg="Executing migration" id="Update folder title length" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.431908454Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=migrator t=2024-09-17T09:10:46.432362616Z level=info msg="migrations completed" performed=485 skipped=0 duration=699.987012ms 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore t=2024-09-17T09:10:46.43278373Z level=info msg="Created default organization" 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=plugin.loader t=2024-09-17T09:10:46.489432444Z level=info msg="Plugin registered" pluginID=input 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=plugin.loader t=2024-09-17T09:10:46.518700653Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=plugin.loader t=2024-09-17T09:10:46.518753712Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-17T09:10:46.591 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=secrets t=2024-09-17T09:10:46.518795019Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-17T09:10:46.592 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=query_data t=2024-09-17T09:10:46.520626078Z level=info msg="Query Service initialization" 2024-09-17T09:10:46.592 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=live.push_http t=2024-09-17T09:10:46.523248514Z level=info msg="Live Push Gateway initialization" 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:45.612505+0000 mgr.y (mgr.25024) 105 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:45.721610+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:45.727922+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:45.729560+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:45.612505+0000 mgr.y (mgr.25024) 105 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:45.721610+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:45.727922+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:45.729560+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:47.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:45.612505+0000 mgr.y (mgr.25024) 105 : cluster [DBG] pgmap v56: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:45.721610+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:45.727922+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:45.729560+0000 mon.c (mon.1) 162 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:47.480 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=infra.usagestats.collector t=2024-09-17T09:10:47.266084695Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-17T09:10:47.480 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=provisioning.datasources t=2024-09-17T09:10:47.321220124Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-17T09:10:47.480 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=provisioning.datasources t=2024-09-17T09:10:47.352783432Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-17T09:10:47.480 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=provisioning.alerting t=2024-09-17T09:10:47.37464599Z level=info msg="starting to provision alerting" 2024-09-17T09:10:47.480 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=provisioning.alerting t=2024-09-17T09:10:47.374664537Z level=info msg="finished to provision alerting" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=grafanaStorageLogger t=2024-09-17T09:10:47.374830375Z level=info msg="storage starting" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=ngalert.state.manager t=2024-09-17T09:10:47.375004011Z level=info msg="Warming state cache for startup" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=http.server t=2024-09-17T09:10:47.376574695Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=ngalert.state.manager t=2024-09-17T09:10:47.385110829Z level=info msg="State cache has been initialized" states=0 duration=10.106653ms 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=ticker t=2024-09-17T09:10:47.38515639Z level=info msg=starting first_tick=2024-09-17T09:10:50Z 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=ngalert.multiorg.alertmanager t=2024-09-17T09:10:47.385146397Z level=info msg="starting MultiOrg Alertmanager" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore.transactions t=2024-09-17T09:10:47.386082293Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore.transactions t=2024-09-17T09:10:47.432410524Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore.transactions t=2024-09-17T09:10:47.442862391Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-17T09:10:47.481 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:10:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore.transactions t=2024-09-17T09:10:47.479851757Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-17T09:10:48.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:47.613072+0000 mgr.y (mgr.25024) 106 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:47.613072+0000 mgr.y (mgr.25024) 106 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:47.613072+0000 mgr.y (mgr.25024) 106 : cluster [DBG] pgmap v57: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 119 B/s rd, 0 op/s 2024-09-17T09:10:50.004 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:10:50.654 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (19s) 14s ago 22m 15.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 starting - - - - 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 12s ago 21m - - 24ecd6d5f14c 68eca5a6c237 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (78s) 12s ago 25m 435M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (11m) 14s ago 27m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (27m) 14s ago 27m 93.9M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (26m) 12s ago 26m 72.1M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (26m) 14s ago 26m 65.2M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (61s) 14s ago 22m 7922k - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (55s) 12s ago 22m 8598k - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (25m) 14s ago 25m 49.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (25m) 14s ago 25m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:10:50.655 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (25m) 14s ago 24m 46.2M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (24m) 14s ago 24m 50.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (24m) 12s ago 24m 53.7M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (24m) 12s ago 24m 47.1M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (23m) 12s ago 23m 48.5M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (23m) 12s ago 23m 52.5M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (35s) 12s ago 22m 31.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (21m) 14s ago 21m 41.5M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:10:50.656 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (21m) 12s ago 21m 42.9M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:10:51.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:49.613921+0000 mgr.y (mgr.25024) 107 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:49.613921+0000 mgr.y (mgr.25024) 107 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:49.613921+0000 mgr.y (mgr.25024) 107 : cluster [DBG] pgmap v58: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:51.054 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:10:51.054 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:10:51.054 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:10:51.055 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:10:51.056 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:10:51.056 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:10:51.056 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:10:51.056 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr" 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:10:51.388 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T09:10:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T09:10:51.389 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:10:51.389 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:10:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:49.991236+0000 mgr.y (mgr.25024) 108 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:50.322903+0000 mgr.y (mgr.25024) 109 : audit [DBG] from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:50.646783+0000 mgr.y (mgr.25024) 110 : audit [DBG] from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:51.054556+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.27:0/1195723051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:49.991236+0000 mgr.y (mgr.25024) 108 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:50.322903+0000 mgr.y (mgr.25024) 109 : audit [DBG] from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:50.646783+0000 mgr.y (mgr.25024) 110 : audit [DBG] from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:51.054556+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.27:0/1195723051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:52.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:49.991236+0000 mgr.y (mgr.25024) 108 : audit [DBG] from='client.15465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:50.322903+0000 mgr.y (mgr.25024) 109 : audit [DBG] from='client.25402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:50.646783+0000 mgr.y (mgr.25024) 110 : audit [DBG] from='client.25408 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.021 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:51 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:51.054556+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.27:0/1195723051' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:52.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:51.387588+0000 mgr.y (mgr.25024) 111 : audit [DBG] from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:52.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:51.614483+0000 mgr.y (mgr.25024) 112 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:52.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:51.795255+0000 mon.b (mon.2) 177 : audit [DBG] from='client.? 172.21.15.105:0/3570131116' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:52.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:52 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:52.121869+0000 mon.b (mon.2) 178 : audit [INF] from='client.? 172.21.15.105:0/737436347' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:51.387588+0000 mgr.y (mgr.25024) 111 : audit [DBG] from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:51.614483+0000 mgr.y (mgr.25024) 112 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:51.795255+0000 mon.b (mon.2) 177 : audit [DBG] from='client.? 172.21.15.105:0/3570131116' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:52.121869+0000 mon.b (mon.2) 178 : audit [INF] from='client.? 172.21.15.105:0/737436347' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:51.387588+0000 mgr.y (mgr.25024) 111 : audit [DBG] from='client.15489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:51.614483+0000 mgr.y (mgr.25024) 112 : cluster [DBG] pgmap v59: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:51.795255+0000 mon.b (mon.2) 177 : audit [DBG] from='client.? 172.21.15.105:0/3570131116' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:10:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:52.121869+0000 mon.b (mon.2) 178 : audit [INF] from='client.? 172.21.15.105:0/737436347' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:10:54.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:53.598753+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:54.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:53.604866+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:54.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:53.614877+0000 mgr.y (mgr.25024) 113 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:54.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:54.110799+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:54.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:54 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:54.116154+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:53.598753+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:53.604866+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:53.614877+0000 mgr.y (mgr.25024) 113 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:54.110799+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:54.116154+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:53.598753+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:53.604866+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:53.614877+0000 mgr.y (mgr.25024) 113 : cluster [DBG] pgmap v60: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:10:55.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:54.110799+0000 mon.a (mon.0) 932 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:55.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:54 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:54.116154+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:57.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:55.615881+0000 mgr.y (mgr.25024) 114 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:57.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:55.615881+0000 mgr.y (mgr.25024) 114 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:57.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:56 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:55.615881+0000 mgr.y (mgr.25024) 114 : cluster [DBG] pgmap v61: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.355552+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.361859+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.365203+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.366121+0000 mon.c (mon.1) 164 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:57.366846+0000 mgr.y (mgr.25024) 115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:57.367431+0000 mgr.y (mgr.25024) 116 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.371586+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.456420+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.456876+0000 mgr.y (mgr.25024) 117 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.495554+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.498321+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.499836+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.501481+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.502914+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.503923+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.504938+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.506269+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.507943+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.509302+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.510549+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.511765+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.513004+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.514114+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.515127+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:57.515722+0000 mgr.y (mgr.25024) 118 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.520293+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.523317+0000 mon.c (mon.1) 181 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.523607+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.527121+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.530342+0000 mon.c (mon.1) 182 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.530639+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.533941+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.536730+0000 mon.c (mon.1) 183 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.536978+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.540056+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:10:58.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.543088+0000 mon.c (mon.1) 184 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.543335+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.543974+0000 mon.c (mon.1) 185 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.544212+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.549897+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.550652+0000 mon.c (mon.1) 186 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.550939+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.551600+0000 mon.c (mon.1) 187 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.551851+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.555198+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.558084+0000 mon.c (mon.1) 188 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.558339+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.558963+0000 mon.c (mon.1) 189 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.559188+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.562382+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.563385+0000 mon.c (mon.1) 190 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.563674+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.564296+0000 mon.c (mon.1) 191 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.564555+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.567931+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.568713+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.568963+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.572200+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.573165+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.573425+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.574032+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.574254+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.574852+0000 mon.c (mon.1) 195 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.575070+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.575653+0000 mon.c (mon.1) 196 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.575867+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.576451+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.576665+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.577255+0000 mon.c (mon.1) 198 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.577478+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.578049+0000 mon.c (mon.1) 199 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.578260+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.581548+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:10:57.582351+0000 mgr.y (mgr.25024) 119 : cephadm [INF] Upgrade: Complete! 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.582699+0000 mon.c (mon.1) 200 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.582934+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.585335+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:10:58.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:58 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:57.586432+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:58.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.355552+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.361859+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.365203+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.366121+0000 mon.c (mon.1) 164 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:57.366846+0000 mgr.y (mgr.25024) 115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:57.367431+0000 mgr.y (mgr.25024) 116 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.371586+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.456420+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.456876+0000 mgr.y (mgr.25024) 117 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.495554+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.498321+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.499836+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.501481+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.502914+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.503923+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.504938+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.355552+0000 mon.a (mon.0) 934 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.361859+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.365203+0000 mon.c (mon.1) 163 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.366121+0000 mon.c (mon.1) 164 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:57.366846+0000 mgr.y (mgr.25024) 115 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:57.367431+0000 mgr.y (mgr.25024) 116 : cluster [DBG] pgmap v62: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.371586+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.456420+0000 mon.c (mon.1) 165 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.456876+0000 mgr.y (mgr.25024) 117 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:10:58.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.495554+0000 mon.c (mon.1) 166 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.498321+0000 mon.c (mon.1) 167 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.499836+0000 mon.c (mon.1) 168 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.501481+0000 mon.c (mon.1) 169 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.502914+0000 mon.c (mon.1) 170 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.503923+0000 mon.c (mon.1) 171 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.504938+0000 mon.c (mon.1) 172 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.506269+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.507943+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.509302+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.510549+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.511765+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.513004+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.514114+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.515127+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:57.515722+0000 mgr.y (mgr.25024) 118 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.520293+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.523317+0000 mon.c (mon.1) 181 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.523607+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.527121+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.530342+0000 mon.c (mon.1) 182 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.530639+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.533941+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.506269+0000 mon.c (mon.1) 173 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.507943+0000 mon.c (mon.1) 174 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.509302+0000 mon.c (mon.1) 175 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.510549+0000 mon.c (mon.1) 176 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.511765+0000 mon.c (mon.1) 177 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.513004+0000 mon.c (mon.1) 178 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.514114+0000 mon.c (mon.1) 179 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.515127+0000 mon.c (mon.1) 180 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:57.515722+0000 mgr.y (mgr.25024) 118 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.520293+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:58.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.523317+0000 mon.c (mon.1) 181 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.523607+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.527121+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.530342+0000 mon.c (mon.1) 182 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.530639+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.533941+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.536730+0000 mon.c (mon.1) 183 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.536978+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.540056+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.543088+0000 mon.c (mon.1) 184 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.543335+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.543974+0000 mon.c (mon.1) 185 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.544212+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.549897+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.550652+0000 mon.c (mon.1) 186 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.550939+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.551600+0000 mon.c (mon.1) 187 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.551851+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.555198+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.558084+0000 mon.c (mon.1) 188 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.558339+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.536730+0000 mon.c (mon.1) 183 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.536978+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.540056+0000 mon.a (mon.0) 943 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.543088+0000 mon.c (mon.1) 184 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.543335+0000 mon.a (mon.0) 944 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.543974+0000 mon.c (mon.1) 185 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.544212+0000 mon.a (mon.0) 945 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.549897+0000 mon.a (mon.0) 946 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.550652+0000 mon.c (mon.1) 186 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.550939+0000 mon.a (mon.0) 947 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.551600+0000 mon.c (mon.1) 187 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.766 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.551851+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.555198+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.558084+0000 mon.c (mon.1) 188 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.558339+0000 mon.a (mon.0) 950 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.558963+0000 mon.c (mon.1) 189 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.559188+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.562382+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.563385+0000 mon.c (mon.1) 190 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.563674+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.564296+0000 mon.c (mon.1) 191 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.564555+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.567931+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.568713+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.568963+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.572200+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.573165+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.573425+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.574032+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.558963+0000 mon.c (mon.1) 189 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.559188+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:10:58.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.562382+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.563385+0000 mon.c (mon.1) 190 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.563674+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.564296+0000 mon.c (mon.1) 191 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.564555+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.567931+0000 mon.a (mon.0) 955 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.568713+0000 mon.c (mon.1) 192 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.568963+0000 mon.a (mon.0) 956 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.572200+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.573165+0000 mon.c (mon.1) 193 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.573425+0000 mon.a (mon.0) 958 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.769 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.574032+0000 mon.c (mon.1) 194 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.574254+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.574852+0000 mon.c (mon.1) 195 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.575070+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.575653+0000 mon.c (mon.1) 196 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.575867+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.576451+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.576665+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.577255+0000 mon.c (mon.1) 198 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.577478+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.578049+0000 mon.c (mon.1) 199 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.578260+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.581548+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:10:57.582351+0000 mgr.y (mgr.25024) 119 : cephadm [INF] Upgrade: Complete! 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.582699+0000 mon.c (mon.1) 200 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.582934+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.585335+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:57.586432+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.574254+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.574852+0000 mon.c (mon.1) 195 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.575070+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.575653+0000 mon.c (mon.1) 196 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.575867+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.576451+0000 mon.c (mon.1) 197 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.576665+0000 mon.a (mon.0) 962 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.577255+0000 mon.c (mon.1) 198 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.577478+0000 mon.a (mon.0) 963 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.578049+0000 mon.c (mon.1) 199 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.578260+0000 mon.a (mon.0) 964 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.581548+0000 mon.a (mon.0) 965 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:10:57.582351+0000 mgr.y (mgr.25024) 119 : cephadm [INF] Upgrade: Complete! 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.582699+0000 mon.c (mon.1) 200 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.582934+0000 mon.a (mon.0) 966 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.585335+0000 mon.a (mon.0) 967 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:10:58.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:57.586432+0000 mon.c (mon.1) 201 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:10:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:58.362519+0000 mon.a (mon.0) 968 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:10:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:58.362569+0000 mon.a (mon.0) 969 : cluster [INF] Cluster is now healthy 2024-09-17T09:10:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:10:59 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:10:58.940784+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:58.362519+0000 mon.a (mon.0) 968 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:58.362569+0000 mon.a (mon.0) 969 : cluster [INF] Cluster is now healthy 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:10:58.940784+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:58.362519+0000 mon.a (mon.0) 968 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:58.362569+0000 mon.a (mon.0) 969 : cluster [INF] Cluster is now healthy 2024-09-17T09:10:59.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:10:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:10:58.940784+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:00.646 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:00 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:10:59.367787+0000 mgr.y (mgr.25024) 120 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:00.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:10:59.367787+0000 mgr.y (mgr.25024) 120 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:00.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:10:59.367787+0000 mgr.y (mgr.25024) 120 : cluster [DBG] pgmap v63: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:02.719 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:01.368660+0000 mgr.y (mgr.25024) 121 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:02.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:01.368660+0000 mgr.y (mgr.25024) 121 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:02.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:02 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:01.368660+0000 mgr.y (mgr.25024) 121 : cluster [DBG] pgmap v64: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 87 B/s rd, 0 op/s 2024-09-17T09:11:03.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.599664+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:03.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.606443+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:03.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.608183+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:03.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.609103+0000 mon.c (mon.1) 203 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:02.609927+0000 mgr.y (mgr.25024) 122 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:02.610499+0000 mgr.y (mgr.25024) 123 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 93 B/s rd, 0 op/s 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.615272+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.730757+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.732115+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.732737+0000 mon.c (mon.1) 206 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:02.733308+0000 mgr.y (mgr.25024) 124 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:02.733723+0000 mgr.y (mgr.25024) 125 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 112 B/s rd, 0 op/s 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:02.733879+0000 mgr.y (mgr.25024) 126 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:03.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:03 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:02.738758+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.599664+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.606443+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.608183+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.609103+0000 mon.c (mon.1) 203 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:02.609927+0000 mgr.y (mgr.25024) 122 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:02.610499+0000 mgr.y (mgr.25024) 123 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 93 B/s rd, 0 op/s 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.615272+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.730757+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.732115+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.732737+0000 mon.c (mon.1) 206 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:02.733308+0000 mgr.y (mgr.25024) 124 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:02.733723+0000 mgr.y (mgr.25024) 125 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 112 B/s rd, 0 op/s 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:02.733879+0000 mgr.y (mgr.25024) 126 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:02.738758+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.599664+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.606443+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.608183+0000 mon.c (mon.1) 202 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.609103+0000 mon.c (mon.1) 203 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:02.609927+0000 mgr.y (mgr.25024) 122 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:02.610499+0000 mgr.y (mgr.25024) 123 : cluster [DBG] pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 93 B/s rd, 0 op/s 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.615272+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.730757+0000 mon.c (mon.1) 204 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.732115+0000 mon.c (mon.1) 205 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.732737+0000 mon.c (mon.1) 206 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:02.733308+0000 mgr.y (mgr.25024) 124 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:02.733723+0000 mgr.y (mgr.25024) 125 : cluster [DBG] pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 112 B/s rd, 0 op/s 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:02.733879+0000 mgr.y (mgr.25024) 126 : cluster [DBG] pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:04.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:02.738758+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:04 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:03.604199+0000 mon.a (mon.0) 975 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:11:05.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:03.604199+0000 mon.a (mon.0) 975 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:11:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:03.604199+0000 mon.a (mon.0) 975 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:11:05.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:04.734704+0000 mgr.y (mgr.25024) 127 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:05.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:05.133159+0000 mon.b (mon.2) 179 : audit [DBG] from='client.? 172.21.15.105:0/633990051' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:05.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:05.446467+0000 mon.b (mon.2) 180 : audit [INF] from='client.? 172.21.15.105:0/4254123787' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:04.734704+0000 mgr.y (mgr.25024) 127 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:05.133159+0000 mon.b (mon.2) 179 : audit [DBG] from='client.? 172.21.15.105:0/633990051' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:05.446467+0000 mon.b (mon.2) 180 : audit [INF] from='client.? 172.21.15.105:0/4254123787' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:04.734704+0000 mgr.y (mgr.25024) 127 : cluster [DBG] pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:05.133159+0000 mon.b (mon.2) 179 : audit [DBG] from='client.? 172.21.15.105:0/633990051' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:05.446467+0000 mon.b (mon.2) 180 : audit [INF] from='client.? 172.21.15.105:0/4254123787' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:06.921 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:06.735275+0000 mgr.y (mgr.25024) 128 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:06.735275+0000 mgr.y (mgr.25024) 128 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:07.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:06 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:06.735275+0000 mgr.y (mgr.25024) 128 : cluster [DBG] pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:09.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:08.735746+0000 mgr.y (mgr.25024) 129 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:09.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:08.735746+0000 mgr.y (mgr.25024) 129 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:09.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:08.735746+0000 mgr.y (mgr.25024) 129 : cluster [DBG] pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:11.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:10.736694+0000 mgr.y (mgr.25024) 130 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 126 B/s rd, 0 op/s 2024-09-17T09:11:11.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:10 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:10.736694+0000 mgr.y (mgr.25024) 130 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 126 B/s rd, 0 op/s 2024-09-17T09:11:11.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:10.736694+0000 mgr.y (mgr.25024) 130 : cluster [DBG] pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 126 B/s rd, 0 op/s 2024-09-17T09:11:13.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:12.737262+0000 mgr.y (mgr.25024) 131 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:11:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:12.737262+0000 mgr.y (mgr.25024) 131 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:11:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:12.737262+0000 mgr.y (mgr.25024) 131 : cluster [DBG] pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:11:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:14.737993+0000 mgr.y (mgr.25024) 132 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:14.737993+0000 mgr.y (mgr.25024) 132 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:14.737993+0000 mgr.y (mgr.25024) 132 : cluster [DBG] pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:17.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:16.738463+0000 mgr.y (mgr.25024) 133 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:16.738463+0000 mgr.y (mgr.25024) 133 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:16.738463+0000 mgr.y (mgr.25024) 133 : cluster [DBG] pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:19.023 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:18.363201+0000 mon.b (mon.2) 181 : audit [DBG] from='client.? 172.21.15.105:0/2558321576' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:19.023 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:18.681166+0000 mon.b (mon.2) 182 : audit [INF] from='client.? 172.21.15.105:0/2159406361' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:19.023 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:18.738882+0000 mgr.y (mgr.25024) 134 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:18.363201+0000 mon.b (mon.2) 181 : audit [DBG] from='client.? 172.21.15.105:0/2558321576' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:18.681166+0000 mon.b (mon.2) 182 : audit [INF] from='client.? 172.21.15.105:0/2159406361' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:18.738882+0000 mgr.y (mgr.25024) 134 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:18.363201+0000 mon.b (mon.2) 181 : audit [DBG] from='client.? 172.21.15.105:0/2558321576' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:18.681166+0000 mon.b (mon.2) 182 : audit [INF] from='client.? 172.21.15.105:0/2159406361' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:19.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:18.738882+0000 mgr.y (mgr.25024) 134 : cluster [DBG] pgmap v75: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:21.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:20.739666+0000 mgr.y (mgr.25024) 135 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:11:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:20.739666+0000 mgr.y (mgr.25024) 135 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:11:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:20.739666+0000 mgr.y (mgr.25024) 135 : cluster [DBG] pgmap v76: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:11:22.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:21.732191+0000 mgr.y (mgr.25024) 136 : audit [DBG] from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:22.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:21.732191+0000 mgr.y (mgr.25024) 136 : audit [DBG] from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:21.732191+0000 mgr.y (mgr.25024) 136 : audit [DBG] from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:22.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-09-17T09:11:22.978 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:22.740065+0000 mgr.y (mgr.25024) 137 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:22.978 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:22.740065+0000 mgr.y (mgr.25024) 137 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:22.740065+0000 mgr.y (mgr.25024) 137 : cluster [DBG] pgmap v77: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:24.432 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:11:24.748 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:24.419143+0000 mon.c (mon.1) 207 : audit [DBG] from='client.? 172.21.15.27:0/1933123983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:24.749 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:24 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:24.419143+0000 mon.c (mon.1) 207 : audit [DBG] from='client.? 172.21.15.27:0/1933123983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:24.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:24.419143+0000 mon.c (mon.1) 207 : audit [DBG] from='client.? 172.21.15.27:0/1933123983' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:25.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-09-17T09:11:25.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:24.740786+0000 mgr.y (mgr.25024) 138 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:25.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:24.740786+0000 mgr.y (mgr.25024) 138 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:25.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:24.740786+0000 mgr.y (mgr.25024) 138 : cluster [DBG] pgmap v78: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:27.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:26.741223+0000 mgr.y (mgr.25024) 139 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:27.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:26 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:26.741223+0000 mgr.y (mgr.25024) 139 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:26.741223+0000 mgr.y (mgr.25024) 139 : cluster [DBG] pgmap v79: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:27.425 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)" 2024-09-17T09:11:28.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:27.411953+0000 mon.b (mon.2) 183 : audit [DBG] from='client.? 172.21.15.27:0/1680649446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:28.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:27.411953+0000 mon.b (mon.2) 183 : audit [DBG] from='client.? 172.21.15.27:0/1680649446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:28.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-09-17T09:11:28.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:27.411953+0000 mon.b (mon.2) 183 : audit [DBG] from='client.? 172.21.15.27:0/1680649446' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:29.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:28.741634+0000 mgr.y (mgr.25024) 140 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:29.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:28.741634+0000 mgr.y (mgr.25024) 140 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:28.741634+0000 mgr.y (mgr.25024) 140 : cluster [DBG] pgmap v80: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:30.126 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:11:30.363 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:30.114130+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.27:0/4098169302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:30.507 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:30.114130+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.27:0/4098169302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:30.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:30.114130+0000 mon.a (mon.0) 976 : audit [DBG] from='client.? 172.21.15.27:0/4098169302' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:30.702 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-09-17T09:11:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:30.742249+0000 mgr.y (mgr.25024) 141 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:30.742249+0000 mgr.y (mgr.25024) 141 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:31.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:30.742249+0000 mgr.y (mgr.25024) 141 : cluster [DBG] pgmap v81: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:31.624322+0000 mon.b (mon.2) 184 : audit [DBG] from='client.? 172.21.15.105:0/2634288127' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:31.946621+0000 mon.b (mon.2) 185 : audit [INF] from='client.? 172.21.15.105:0/605249291' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:31.624322+0000 mon.b (mon.2) 184 : audit [DBG] from='client.? 172.21.15.105:0/2634288127' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:32.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:32 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:31.946621+0000 mon.b (mon.2) 185 : audit [INF] from='client.? 172.21.15.105:0/605249291' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:31.624322+0000 mon.b (mon.2) 184 : audit [DBG] from='client.? 172.21.15.105:0/2634288127' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:31.946621+0000 mon.b (mon.2) 185 : audit [INF] from='client.? 172.21.15.105:0/605249291' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:33.382 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:32.742582+0000 mgr.y (mgr.25024) 142 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:33.383 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:32.762554+0000 mgr.y (mgr.25024) 143 : audit [DBG] from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:33.383 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:32.742582+0000 mgr.y (mgr.25024) 142 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:33.383 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:32.762554+0000 mgr.y (mgr.25024) 143 : audit [DBG] from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:33.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:32.742582+0000 mgr.y (mgr.25024) 142 : cluster [DBG] pgmap v82: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:33.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:32.762554+0000 mgr.y (mgr.25024) 143 : audit [DBG] from='client.25525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:34.072 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:11:34.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:11:35.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:34.743207+0000 mgr.y (mgr.25024) 144 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:34.743207+0000 mgr.y (mgr.25024) 144 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:34.743207+0000 mgr.y (mgr.25024) 144 : cluster [DBG] pgmap v83: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:11:36.657 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:11:36.658 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:11:36.658 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:11:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:36.655286+0000 mgr.y (mgr.25024) 145 : audit [DBG] from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:37.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:36.743700+0000 mgr.y (mgr.25024) 146 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:36.655286+0000 mgr.y (mgr.25024) 145 : audit [DBG] from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:37.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:36.743700+0000 mgr.y (mgr.25024) 146 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:36.655286+0000 mgr.y (mgr.25024) 145 : audit [DBG] from='client.25531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:36.743700+0000 mgr.y (mgr.25024) 146 : cluster [DBG] pgmap v84: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:37.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:11:38.956 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:38.744057+0000 mgr.y (mgr.25024) 147 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:38.957 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:38.744057+0000 mgr.y (mgr.25024) 147 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:38.744057+0000 mgr.y (mgr.25024) 147 : cluster [DBG] pgmap v85: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:39.534 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:11:39.534 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:11:39.534 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:11:40.191 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:39.533561+0000 mon.c (mon.1) 208 : audit [DBG] from='client.? 172.21.15.27:0/3890330218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:11:40.191 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:39.533561+0000 mon.c (mon.1) 208 : audit [DBG] from='client.? 172.21.15.27:0/3890330218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:11:40.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-09-17T09:11:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:39.533561+0000 mon.c (mon.1) 208 : audit [DBG] from='client.? 172.21.15.27:0/3890330218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:11:41.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:40.744747+0000 mgr.y (mgr.25024) 148 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:41.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:40.744747+0000 mgr.y (mgr.25024) 148 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:40 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:40.744747+0000 mgr.y (mgr.25024) 148 : cluster [DBG] pgmap v86: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:43.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:42.200507+0000 mgr.y (mgr.25024) 149 : audit [DBG] from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:42.550009+0000 mgr.y (mgr.25024) 150 : audit [DBG] from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.143 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:42.745112+0000 mgr.y (mgr.25024) 151 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:43.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:42.200507+0000 mgr.y (mgr.25024) 149 : audit [DBG] from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.143 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:42.550009+0000 mgr.y (mgr.25024) 150 : audit [DBG] from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.144 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:42.745112+0000 mgr.y (mgr.25024) 151 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:42.200507+0000 mgr.y (mgr.25024) 149 : audit [DBG] from='client.15609 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:42.550009+0000 mgr.y (mgr.25024) 150 : audit [DBG] from='client.25549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mon", "hosts": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:42.745112+0000 mgr.y (mgr.25024) 151 : cluster [DBG] pgmap v87: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:43.986 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:44.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:11:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:43.979254+0000 mgr.y (mgr.25024) 152 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:43.983954+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:43.986553+0000 mon.c (mon.1) 209 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:44.367607+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:44.368313+0000 mon.c (mon.1) 211 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:44.368908+0000 mgr.y (mgr.25024) 153 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:44.369359+0000 mgr.y (mgr.25024) 154 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 B/s rd, 0 op/s 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:44.369554+0000 mgr.y (mgr.25024) 155 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:44.375137+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:44.484831+0000 mgr.y (mgr.25024) 156 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:44.691741+0000 mon.b (mon.2) 186 : audit [DBG] from='client.? 172.21.15.105:0/3179435503' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:43.979254+0000 mgr.y (mgr.25024) 152 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:43.983954+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:43.986553+0000 mon.c (mon.1) 209 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:45.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:44.367607+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:44.368313+0000 mon.c (mon.1) 211 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:44.368908+0000 mgr.y (mgr.25024) 153 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:44.369359+0000 mgr.y (mgr.25024) 154 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 B/s rd, 0 op/s 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:44.369554+0000 mgr.y (mgr.25024) 155 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:44.375137+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:44.484831+0000 mgr.y (mgr.25024) 156 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:44.691741+0000 mon.b (mon.2) 186 : audit [DBG] from='client.? 172.21.15.105:0/3179435503' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:43.979254+0000 mgr.y (mgr.25024) 152 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:43.983954+0000 mon.a (mon.0) 977 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:43.986553+0000 mon.c (mon.1) 209 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:44.367607+0000 mon.c (mon.1) 210 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:44.368313+0000 mon.c (mon.1) 211 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:11:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:44.368908+0000 mgr.y (mgr.25024) 153 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:11:45.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:44.369359+0000 mgr.y (mgr.25024) 154 : cluster [DBG] pgmap v88: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 88 B/s rd, 0 op/s 2024-09-17T09:11:45.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:44.369554+0000 mgr.y (mgr.25024) 155 : cluster [DBG] pgmap v89: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:45.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:44.375137+0000 mon.a (mon.0) 978 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:45.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:44.484831+0000 mgr.y (mgr.25024) 156 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:11:45.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:44.691741+0000 mon.b (mon.2) 186 : audit [DBG] from='client.? 172.21.15.105:0/3179435503' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:45.006950+0000 mon.b (mon.2) 187 : audit [INF] from='client.? 172.21.15.105:0/2886674191' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:46.210 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:45.006950+0000 mon.b (mon.2) 187 : audit [INF] from='client.? 172.21.15.105:0/2886674191' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:46.210 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:45.006950+0000 mon.b (mon.2) 187 : audit [INF] from='client.? 172.21.15.105:0/2886674191' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: access denied 2024-09-17T09:11:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:46.370250+0000 mgr.y (mgr.25024) 157 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:47.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:46.370250+0000 mgr.y (mgr.25024) 157 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:46.370250+0000 mgr.y (mgr.25024) 157 : cluster [DBG] pgmap v90: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:47.616 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:11:48.271 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:11:48.271 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (77s) 72s ago 22m 15.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:11:48.271 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (63s) 45s ago 22m 80.3M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:11:48.271 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 45s ago 22m - - 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (2m) 45s ago 26m 435M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (12m) 72s ago 28m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (28m) 72s ago 28m 93.9M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (27m) 45s ago 27m 73.0M 2048M 16.2.0 24ecd6d5f14c 87738d8bcab9 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (27m) 72s ago 27m 65.2M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (119s) 72s ago 23m 7922k - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (112s) 45s ago 23m 15.5M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (26m) 72s ago 26m 49.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (26m) 72s ago 26m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (25m) 72s ago 25m 46.2M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (25m) 72s ago 25m 50.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:11:48.272 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (25m) 45s ago 25m 53.7M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (25m) 45s ago 25m 47.2M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (24m) 45s ago 24m 48.6M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (24m) 45s ago 24m 52.6M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (93s) 45s ago 23m 37.5M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (22m) 72s ago 22m 41.5M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:11:48.273 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (22m) 45s ago 22m 43.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:11:48.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.170394+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:47.171020+0000 mgr.y (mgr.25024) 158 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:11:48.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:47.171076+0000 mgr.y (mgr.25024) 159 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.172568+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.180010+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.181594+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:47.182215+0000 mgr.y (mgr.25024) 160 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.188731+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.190002+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.190699+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:47.191191+0000 mgr.y (mgr.25024) 161 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-17T09:11:48.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.602742+0000 mgr.y (mgr.25024) 162 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.170394+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:47.171020+0000 mgr.y (mgr.25024) 158 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:11:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:47.171076+0000 mgr.y (mgr.25024) 159 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:11:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.172568+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.180010+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.181594+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:47.182215+0000 mgr.y (mgr.25024) 160 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.188731+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.190002+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.190699+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:47.191191+0000 mgr.y (mgr.25024) 161 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.602742+0000 mgr.y (mgr.25024) 162 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.170394+0000 mon.a (mon.0) 979 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:47.171020+0000 mgr.y (mgr.25024) 158 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:11:48.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:47.171076+0000 mgr.y (mgr.25024) 159 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.172568+0000 mon.c (mon.1) 212 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.180010+0000 mon.a (mon.0) 980 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.181594+0000 mon.c (mon.1) 213 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:47.182215+0000 mgr.y (mgr.25024) 160 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.188731+0000 mon.a (mon.0) 981 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.190002+0000 mon.c (mon.1) 214 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.190699+0000 mon.c (mon.1) 215 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:47.191191+0000 mgr.y (mgr.25024) 161 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-17T09:11:48.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.602742+0000 mgr.y (mgr.25024) 162 : audit [DBG] from='client.15645 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:11:48.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 13, 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:11:48.668 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi105", 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/1 daemons upgraded", 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T09:11:48.999 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:11:49.000 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:11:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:47.939200+0000 mgr.y (mgr.25024) 163 : audit [DBG] from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.263188+0000 mgr.y (mgr.25024) 164 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cluster 2024-09-17T09:11:48.370670+0000 mgr.y (mgr.25024) 165 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:48.428693+0000 mgr.y (mgr.25024) 166 : cephadm [INF] Upgrade: Updating mon.b 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.434327+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.436973+0000 mon.c (mon.1) 216 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.437604+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.438195+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: cephadm 2024-09-17T09:11:48.439161+0000 mgr.y (mgr.25024) 167 : cephadm [INF] Deploying daemon mon.b on smithi105 2024-09-17T09:11:49.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: audit 2024-09-17T09:11:48.666612+0000 mon.c (mon.1) 219 : audit [DBG] from='client.? 172.21.15.27:0/811311818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:47.939200+0000 mgr.y (mgr.25024) 163 : audit [DBG] from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.263188+0000 mgr.y (mgr.25024) 164 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:48.370670+0000 mgr.y (mgr.25024) 165 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:48.428693+0000 mgr.y (mgr.25024) 166 : cephadm [INF] Upgrade: Updating mon.b 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.434327+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.436973+0000 mon.c (mon.1) 216 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.437604+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:11:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.438195+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:11:48.439161+0000 mgr.y (mgr.25024) 167 : cephadm [INF] Deploying daemon mon.b on smithi105 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:48.666612+0000 mon.c (mon.1) 219 : audit [DBG] from='client.? 172.21.15.27:0/811311818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:47.939200+0000 mgr.y (mgr.25024) 163 : audit [DBG] from='client.15651 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.263188+0000 mgr.y (mgr.25024) 164 : audit [DBG] from='client.15657 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:48.370670+0000 mgr.y (mgr.25024) 165 : cluster [DBG] pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:48.428693+0000 mgr.y (mgr.25024) 166 : cephadm [INF] Upgrade: Updating mon.b 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.434327+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.436973+0000 mon.c (mon.1) 216 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.437604+0000 mon.c (mon.1) 217 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:11:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.438195+0000 mon.c (mon.1) 218 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:11:49.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:11:48.439161+0000 mgr.y (mgr.25024) 167 : cephadm [INF] Deploying daemon mon.b on smithi105 2024-09-17T09:11:49.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:48.666612+0000 mon.c (mon.1) 219 : audit [DBG] from='client.? 172.21.15.27:0/811311818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:11:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 systemd[1]: Stopping Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:11:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T09:11:50.252+0000 7f3b2fa7f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-17T09:11:50.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b[23870]: debug 2024-09-17T09:11:50.252+0000 7f3b2fa7f700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-09-17T09:11:50.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 podman[70294]: 2024-09-17 09:11:50.314020169 +0000 UTC m=+0.183309646 container died 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2) 2024-09-17T09:11:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 podman[70294]: 2024-09-17 09:11:50.735510024 +0000 UTC m=+0.604799506 container cleanup 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, RELEASE=HEAD, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2) 2024-09-17T09:11:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 bash[70294]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b 2024-09-17T09:11:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:50 smithi105 podman[70307]: 2024-09-17 09:11:50.953735604 +0000 UTC m=+0.635366506 container remove 87738d8bcab9164c27d62076dc3ac8b326df5115e323664a2231fcd029d64e90 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Dimitri Savineau , org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, ceph=True, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:11:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-17T09:11:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 systemd[1]: Stopped Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:11:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 systemd[1]: Starting Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 podman[70448]: 2024-09-17 09:11:51.511962131 +0000 UTC m=+0.015862744 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 podman[70448]: 2024-09-17 09:11:51.630219793 +0000 UTC m=+0.134120398 container create 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 podman[70448]: 2024-09-17 09:11:51.857854142 +0000 UTC m=+0.361754752 container init 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T09:11:51.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 podman[70448]: 2024-09-17 09:11:51.861231356 +0000 UTC m=+0.365131952 container start 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-17T09:11:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable), process ceph-mon, pid 2 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: pidfile_write: ignore empty --pid-file 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: load: jerasure load: lrc 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: RocksDB version: 7.9.2 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Git sha 0 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Compile date 2024-09-16 15:15:13 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: DB SUMMARY 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: DB Session ID: YMAV5V08JJY1LLZIRGKE 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:11:52.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: MANIFEST file: MANIFEST-000005 size: 2642 Bytes 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000047.sst 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000045.log size: 1527410 ; 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.env: 0x5650f5d02bc0 2024-09-17T09:11:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.info_log: 0x5650f7944be0 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.statistics: (nil) 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.use_fsync: 0 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:11:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.db_log_dir: 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.wal_dir: 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:11:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.write_buffer_manager: 0x5650f7952b40 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:11:52.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.unordered_write: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.row_cache: None 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.wal_filter: None 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:11:52.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.wal_compression: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:11:52.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:11:52.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_open_files: -1 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Compression algorithms supported: 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kZSTD supported: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:11:52.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.merge_operator: 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_filter: None 2024-09-17T09:11:52.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5650f79447e0) 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_type: 0 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_index_type: 0 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_shortening: 1 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:11:52.290 INFO:journalctl@ceph.mon.b.smithi105.stdout: checksum: 4 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: no_block_cache: 0 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache: 0x5650f7939350 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_options: 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: capacity : 536870912 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_shard_bits : 4 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: strict_capacity_limit : 0 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_compressed: (nil) 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: persistent_cache: (nil) 2024-09-17T09:11:52.291 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size: 4096 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size_deviation: 10 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_restart_interval: 16 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_block_restart_interval: 1 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: metadata_block_size: 4096 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: partition_filters: 0 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: use_delta_encoding: 1 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: filter_policy: bloomfilter 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: whole_key_filtering: 1 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: verify_compression: 0 2024-09-17T09:11:52.292 INFO:journalctl@ceph.mon.b.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: format_version: 5 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_index_compression: 1 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_align: 0 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_auto_readahead_size: 262144 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: prepopulate_block_cache: 0 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: initial_auto_readahead_size: 8192 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:11:52.293 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression: NoCompression 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.num_levels: 7 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:11:52.294 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:11:52.295 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:11:52.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:11:52.297 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:11:52.298 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:11:52.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.ttl: 2592000 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:11:52.300 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 47.sst 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 49, last_sequence is 27377, log_number is 45,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:11:52.301 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 45 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c68d90e8-3a55-47c0-b65e-b7e6fe1f5ef0 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564311887283, "job": 1, "event": "recovery_started", "wal_files": [45]} 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #45 mode 2 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564311891404, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 50, "file_size": 1014190, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 27372, "largest_seqno": 27646, "table_properties": {"data_size": 1011859, "index_size": 1037, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2224, "raw_average_key_size": 23, "raw_value_size": 1009166, "raw_average_value_size": 10512, "num_data_blocks": 46, "num_entries": 96, "num_filter_entries": 96, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726564311, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c68d90e8-3a55-47c0-b65e-b7e6fe1f5ef0", "db_session_id": "YMAV5V08JJY1LLZIRGKE", "orig_file_number": 50, "seqno_to_time_mapping": "N/A"}} 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564311891511, "job": 1, "event": "recovery_finished"} 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/version_set.cc:5047] Creating manifest 52 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000045.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5650f7964e00 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: DB pointer 0x5650f7a58000 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T09:11:52.302 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** DB Stats ** 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T09:11:52.303 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 990.42 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 429.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: L6 1/0 30.92 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 2/0 31.89 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 429.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 429.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 429.7 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.304 INFO:journalctl@ceph.mon.b.smithi105.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.00 GB write, 149.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.00 GB write, 149.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:11:52.305 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache BinnedLRUCache@0x5650f7939350#2 capacity: 512.00 MB usage: 1.52 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.20 KB,0.000229478%) Misc(1,0.00 KB,0%) 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: starting mon.b rank 2 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???) e3 preinit fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).mds e1 new map 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).mds e1 print_map 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: e1 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:11:52.306 INFO:journalctl@ceph.mon.b.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout: legacy client fscid: -1 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout: No filesystems configured 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 ceph-mon[70467]: mon.b@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 bash[70448]: 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b 2024-09-17T09:11:52.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:11:51 smithi105 systemd[1]: Started Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:11:53.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:50.371727+0000 mgr.y (mgr.25024) 169 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:53.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.103025+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2024-09-17T09:11:53.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.103621+0000 mon.c (mon.1) 220 : cluster [INF] mon.c calling monitor election 2024-09-17T09:11:53.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.103684+0000 mon.a (mon.0) 983 : cluster [INF] mon.a calling monitor election 2024-09-17T09:11:53.365 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.105974+0000 mon.a (mon.0) 984 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.111116+0000 mon.a (mon.0) 985 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.111187+0000 mon.a (mon.0) 986 : cluster [DBG] fsmap 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.111221+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.112016+0000 mon.a (mon.0) 988 : cluster [DBG] mgrmap e36: y(active, since 2m), standbys: x 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.116106+0000 mon.a (mon.0) 989 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.116136+0000 mon.a (mon.0) 990 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.116153+0000 mon.a (mon.0) 991 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:52.119220+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:52.125812+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:52.127770+0000 mon.c (mon.1) 221 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:53.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:52.372200+0000 mgr.y (mgr.25024) 170 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:50.371727+0000 mgr.y (mgr.25024) 169 : cluster [DBG] pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.103025+0000 mon.b (mon.2) 1 : cluster [INF] mon.b calling monitor election 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.103621+0000 mon.c (mon.1) 220 : cluster [INF] mon.c calling monitor election 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.103684+0000 mon.a (mon.0) 983 : cluster [INF] mon.a calling monitor election 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.105974+0000 mon.a (mon.0) 984 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.111116+0000 mon.a (mon.0) 985 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.111187+0000 mon.a (mon.0) 986 : cluster [DBG] fsmap 2024-09-17T09:11:53.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.111221+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.112016+0000 mon.a (mon.0) 988 : cluster [DBG] mgrmap e36: y(active, since 2m), standbys: x 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.116106+0000 mon.a (mon.0) 989 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.116136+0000 mon.a (mon.0) 990 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.116153+0000 mon.a (mon.0) 991 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:52.119220+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:52.125812+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:52.127770+0000 mon.c (mon.1) 221 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:11:53.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:52.372200+0000 mgr.y (mgr.25024) 170 : cluster [DBG] pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 106 B/s rd, 0 op/s 2024-09-17T09:11:55.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:54.372552+0000 mgr.y (mgr.25024) 171 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:11:55.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:54.372552+0000 mgr.y (mgr.25024) 171 : cluster [DBG] pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-09-17T09:11:57.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:11:56 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:11:56.901+0000 7fca5c20c640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-17T09:11:57.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:56.373391+0000 mgr.y (mgr.25024) 172 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:57.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:56.373391+0000 mgr.y (mgr.25024) 172 : cluster [DBG] pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:11:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:58.050878+0000 mon.b (mon.2) 2 : audit [DBG] from='client.44114 172.21.15.105:0/782025259' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:11:58.373811+0000 mgr.y (mgr.25024) 173 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:58.374539+0000 mon.b (mon.2) 3 : audit [INF] from='client.44120 172.21.15.105:0/210968640' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:11:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:58.050878+0000 mon.b (mon.2) 2 : audit [DBG] from='client.44114 172.21.15.105:0/782025259' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:11:59.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:11:58.373811+0000 mgr.y (mgr.25024) 173 : cluster [DBG] pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:11:59.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:11:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:58.374539+0000 mon.b (mon.2) 3 : audit [INF] from='client.44120 172.21.15.105:0/210968640' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:59.012116+0000 mon.a (mon.0) 994 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:11:59.020885+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:59.012116+0000 mon.a (mon.0) 994 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:11:59.020885+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:00.371309+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:00.374796+0000 mgr.y (mgr.25024) 174 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:00.380309+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:00.371309+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:00.374796+0000 mgr.y (mgr.25024) 174 : cluster [DBG] pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:01.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:01 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:00.380309+0000 mon.a (mon.0) 997 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:03.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:02.375338+0000 mgr.y (mgr.25024) 175 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:03.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:02.375338+0000 mgr.y (mgr.25024) 175 : cluster [DBG] pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.499939+0000 mgr.y (mgr.25024) 176 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T09:12:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.505431+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.512704+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.514115+0000 mon.c (mon.1) 222 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.515117+0000 mon.c (mon.1) 223 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.515949+0000 mgr.y (mgr.25024) 177 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.516507+0000 mgr.y (mgr.25024) 178 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 91 B/s rd, 0 op/s 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.516769+0000 mgr.y (mgr.25024) 179 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 111 B/s rd, 0 op/s 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.521400+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.642210+0000 mon.c (mon.1) 224 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.644833+0000 mon.c (mon.1) 225 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.646274+0000 mon.c (mon.1) 226 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.646884+0000 mgr.y (mgr.25024) 180 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.653696+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.655328+0000 mon.c (mon.1) 227 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.655960+0000 mgr.y (mgr.25024) 181 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-17T09:12:04.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.660061+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.499939+0000 mgr.y (mgr.25024) 176 : cephadm [INF] Detected new or changed devices on smithi105 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.505431+0000 mon.a (mon.0) 998 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.512704+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.514115+0000 mon.c (mon.1) 222 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.515117+0000 mon.c (mon.1) 223 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.515949+0000 mgr.y (mgr.25024) 177 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.516507+0000 mgr.y (mgr.25024) 178 : cluster [DBG] pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 91 B/s rd, 0 op/s 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.516769+0000 mgr.y (mgr.25024) 179 : cluster [DBG] pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 111 B/s rd, 0 op/s 2024-09-17T09:12:04.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.521400+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.642210+0000 mon.c (mon.1) 224 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.644833+0000 mon.c (mon.1) 225 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.646274+0000 mon.c (mon.1) 226 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.646884+0000 mgr.y (mgr.25024) 180 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.653696+0000 mon.a (mon.0) 1001 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.655328+0000 mon.c (mon.1) 227 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.655960+0000 mgr.y (mgr.25024) 181 : cephadm [INF] Upgrade: Setting container_image for all mds 2024-09-17T09:12:04.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.660061+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.661967+0000 mon.c (mon.1) 228 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.662562+0000 mgr.y (mgr.25024) 182 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.666431+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.667601+0000 mon.c (mon.1) 229 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.668166+0000 mgr.y (mgr.25024) 183 : cephadm [INF] Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.673336+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.674278+0000 mon.c (mon.1) 230 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.674572+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.678013+0000 mon.a (mon.0) 1006 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.679217+0000 mon.c (mon.1) 231 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.679786+0000 mgr.y (mgr.25024) 184 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.683639+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.684993+0000 mon.c (mon.1) 232 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.685572+0000 mgr.y (mgr.25024) 185 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.689472+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.691077+0000 mon.c (mon.1) 233 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.691654+0000 mgr.y (mgr.25024) 186 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.695523+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.661967+0000 mon.c (mon.1) 228 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.763 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.662562+0000 mgr.y (mgr.25024) 182 : cephadm [INF] Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.666431+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.667601+0000 mon.c (mon.1) 229 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.668166+0000 mgr.y (mgr.25024) 183 : cephadm [INF] Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.673336+0000 mon.a (mon.0) 1004 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.674278+0000 mon.c (mon.1) 230 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.674572+0000 mon.a (mon.0) 1005 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.678013+0000 mon.a (mon.0) 1006 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.679217+0000 mon.c (mon.1) 231 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.679786+0000 mgr.y (mgr.25024) 184 : cephadm [INF] Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.683639+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.764 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.684993+0000 mon.c (mon.1) 232 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.685572+0000 mgr.y (mgr.25024) 185 : cephadm [INF] Upgrade: Setting container_image for all nfs 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.689472+0000 mon.a (mon.0) 1008 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.691077+0000 mon.c (mon.1) 233 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.691654+0000 mgr.y (mgr.25024) 186 : cephadm [INF] Upgrade: Setting container_image for all nvmeof 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.695523+0000 mon.a (mon.0) 1009 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.696886+0000 mon.c (mon.1) 234 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.697428+0000 mgr.y (mgr.25024) 187 : cephadm [INF] Upgrade: Setting container_image for all node-exporter 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.698508+0000 mon.c (mon.1) 235 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.698711+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.699761+0000 mon.c (mon.1) 236 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.700278+0000 mgr.y (mgr.25024) 188 : cephadm [INF] Upgrade: Setting container_image for all prometheus 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.701283+0000 mon.c (mon.1) 237 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.701525+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.702537+0000 mon.c (mon.1) 238 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.703053+0000 mgr.y (mgr.25024) 189 : cephadm [INF] Upgrade: Setting container_image for all alertmanager 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.704098+0000 mon.c (mon.1) 239 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.704304+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.705330+0000 mon.c (mon.1) 240 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.705843+0000 mgr.y (mgr.25024) 190 : cephadm [INF] Upgrade: Setting container_image for all grafana 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.706830+0000 mon.c (mon.1) 241 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.707041+0000 mon.a (mon.0) 1013 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.696886+0000 mon.c (mon.1) 234 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.697428+0000 mgr.y (mgr.25024) 187 : cephadm [INF] Upgrade: Setting container_image for all node-exporter 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.698508+0000 mon.c (mon.1) 235 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.698711+0000 mon.a (mon.0) 1010 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.699761+0000 mon.c (mon.1) 236 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.700278+0000 mgr.y (mgr.25024) 188 : cephadm [INF] Upgrade: Setting container_image for all prometheus 2024-09-17T09:12:04.767 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.701283+0000 mon.c (mon.1) 237 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.701525+0000 mon.a (mon.0) 1011 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.702537+0000 mon.c (mon.1) 238 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.703053+0000 mgr.y (mgr.25024) 189 : cephadm [INF] Upgrade: Setting container_image for all alertmanager 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.704098+0000 mon.c (mon.1) 239 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.704304+0000 mon.a (mon.0) 1012 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.705330+0000 mon.c (mon.1) 240 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.705843+0000 mgr.y (mgr.25024) 190 : cephadm [INF] Upgrade: Setting container_image for all grafana 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.706830+0000 mon.c (mon.1) 241 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.707041+0000 mon.a (mon.0) 1013 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.707944+0000 mon.c (mon.1) 242 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.708477+0000 mgr.y (mgr.25024) 191 : cephadm [INF] Upgrade: Setting container_image for all loki 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.709500+0000 mon.c (mon.1) 243 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.709716+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.710622+0000 mon.c (mon.1) 244 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.711116+0000 mgr.y (mgr.25024) 192 : cephadm [INF] Upgrade: Setting container_image for all promtail 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.712129+0000 mon.c (mon.1) 245 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.712366+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.712882+0000 mgr.y (mgr.25024) 193 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.713911+0000 mon.c (mon.1) 246 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.714113+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:12:04.769 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.717710+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.718568+0000 mon.c (mon.1) 247 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.718870+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.722418+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.723115+0000 mon.c (mon.1) 248 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.707944+0000 mon.c (mon.1) 242 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.708477+0000 mgr.y (mgr.25024) 191 : cephadm [INF] Upgrade: Setting container_image for all loki 2024-09-17T09:12:04.770 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.709500+0000 mon.c (mon.1) 243 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.709716+0000 mon.a (mon.0) 1014 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.710622+0000 mon.c (mon.1) 244 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.711116+0000 mgr.y (mgr.25024) 192 : cephadm [INF] Upgrade: Setting container_image for all promtail 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.712129+0000 mon.c (mon.1) 245 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.712366+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.712882+0000 mgr.y (mgr.25024) 193 : cephadm [INF] Upgrade: Finalizing container_image settings 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.713911+0000 mon.c (mon.1) 246 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.714113+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.717710+0000 mon.a (mon.0) 1017 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.718568+0000 mon.c (mon.1) 247 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.718870+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.771 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.722418+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.723115+0000 mon.c (mon.1) 248 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.723360+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.726706+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.727723+0000 mon.c (mon.1) 249 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.727967+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.728565+0000 mon.c (mon.1) 250 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.728769+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:12:04.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.731967+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.732869+0000 mon.c (mon.1) 251 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.733113+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.733731+0000 mon.c (mon.1) 252 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.733952+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.737299+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:12:04.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.738233+0000 mon.c (mon.1) 253 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.738477+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.723360+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.726706+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.727723+0000 mon.c (mon.1) 249 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.727967+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.728565+0000 mon.c (mon.1) 250 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.728769+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:12:04.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.731967+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.732869+0000 mon.c (mon.1) 251 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.733113+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.733731+0000 mon.c (mon.1) 252 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.733952+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.737299+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.738233+0000 mon.c (mon.1) 253 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.775 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.738477+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.739079+0000 mon.c (mon.1) 254 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.739290+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.742708+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.743584+0000 mon.c (mon.1) 255 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.743867+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.744530+0000 mon.c (mon.1) 256 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.744791+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.748160+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.749043+0000 mon.c (mon.1) 257 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:12:04.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.749286+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.752469+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.753278+0000 mon.c (mon.1) 258 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.753549+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.754132+0000 mon.c (mon.1) 259 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.754353+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.754959+0000 mon.c (mon.1) 260 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.755167+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.755745+0000 mon.c (mon.1) 261 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.755961+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.756550+0000 mon.c (mon.1) 262 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.756766+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.757339+0000 mon.c (mon.1) 263 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.757558+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.758462+0000 mon.c (mon.1) 264 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.758996+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.762624+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.763266+0000 mgr.y (mgr.25024) 194 : cephadm [INF] Upgrade: Complete! 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.763648+0000 mon.c (mon.1) 265 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.763936+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.766560+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.739079+0000 mon.c (mon.1) 254 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.739290+0000 mon.a (mon.0) 1029 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.742708+0000 mon.a (mon.0) 1030 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:12:04.778 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.743584+0000 mon.c (mon.1) 255 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.743867+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.744530+0000 mon.c (mon.1) 256 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.744791+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.748160+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.749043+0000 mon.c (mon.1) 257 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.749286+0000 mon.a (mon.0) 1034 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.752469+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.753278+0000 mon.c (mon.1) 258 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.753549+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.754132+0000 mon.c (mon.1) 259 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.779 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.754353+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.767176+0000 mon.c (mon.1) 266 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.768384+0000 mon.c (mon.1) 267 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.768990+0000 mon.c (mon.1) 268 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.769528+0000 mgr.y (mgr.25024) 195 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.769946+0000 mgr.y (mgr.25024) 196 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 138 B/s rd, 0 op/s 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.770096+0000 mgr.y (mgr.25024) 197 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 189 B/s rd, 0 op/s 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.774279+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.880805+0000 mon.c (mon.1) 269 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.882052+0000 mon.c (mon.1) 270 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.882648+0000 mon.c (mon.1) 271 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.887947+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.780 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:03.949536+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.781 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.754959+0000 mon.c (mon.1) 260 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.781 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.755167+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.781 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.755745+0000 mon.c (mon.1) 261 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.781 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.755961+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.781 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.756550+0000 mon.c (mon.1) 262 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.756766+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.757339+0000 mon.c (mon.1) 263 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.757558+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.758462+0000 mon.c (mon.1) 264 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.758996+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.762624+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.763266+0000 mgr.y (mgr.25024) 194 : cephadm [INF] Upgrade: Complete! 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.763648+0000 mon.c (mon.1) 265 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.763936+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.766560+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25024 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.767176+0000 mon.c (mon.1) 266 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.768384+0000 mon.c (mon.1) 267 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.782 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.768990+0000 mon.c (mon.1) 268 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.769528+0000 mgr.y (mgr.25024) 195 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.769946+0000 mgr.y (mgr.25024) 196 : cluster [DBG] pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 138 B/s rd, 0 op/s 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.770096+0000 mgr.y (mgr.25024) 197 : cluster [DBG] pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 189 B/s rd, 0 op/s 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.774279+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.880805+0000 mon.c (mon.1) 269 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.882052+0000 mon.c (mon.1) 270 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.882648+0000 mon.c (mon.1) 271 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.887947+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:04.783 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:04 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:03.949536+0000 mon.a (mon.0) 1048 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:03.883227+0000 mgr.y (mgr.25024) 198 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.883604+0000 mgr.y (mgr.25024) 199 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:03.883789+0000 mgr.y (mgr.25024) 200 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:03.883227+0000 mgr.y (mgr.25024) 198 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.883604+0000 mgr.y (mgr.25024) 199 : cluster [DBG] pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:06.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:05 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:03.883789+0000 mgr.y (mgr.25024) 200 : cluster [DBG] pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:05.884570+0000 mgr.y (mgr.25024) 201 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:05.884570+0000 mgr.y (mgr.25024) 201 : cluster [DBG] pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:10.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:07.884975+0000 mgr.y (mgr.25024) 202 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:07.884975+0000 mgr.y (mgr.25024) 202 : cluster [DBG] pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:09.886001+0000 mgr.y (mgr.25024) 203 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:09.886001+0000 mgr.y (mgr.25024) 203 : cluster [DBG] pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:13.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:11.789639+0000 mon.b (mon.2) 4 : audit [DBG] from='client.44138 172.21.15.105:0/2364028387' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:13.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:12.108042+0000 mon.b (mon.2) 5 : audit [INF] from='client.44144 172.21.15.105:0/4060081314' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:13.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:11.789639+0000 mon.b (mon.2) 4 : audit [DBG] from='client.44138 172.21.15.105:0/2364028387' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:13.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:12 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:12.108042+0000 mon.b (mon.2) 5 : audit [INF] from='client.44144 172.21.15.105:0/4060081314' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:11.886427+0000 mgr.y (mgr.25024) 204 : cluster [DBG] pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:13 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:11.886427+0000 mgr.y (mgr.25024) 204 : cluster [DBG] pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:16.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:13.886867+0000 mgr.y (mgr.25024) 205 : cluster [DBG] pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:16.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:15 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:13.886867+0000 mgr.y (mgr.25024) 205 : cluster [DBG] pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:18.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:15.887681+0000 mgr.y (mgr.25024) 206 : cluster [DBG] pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:17 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:15.887681+0000 mgr.y (mgr.25024) 206 : cluster [DBG] pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:19.678 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:17.888254+0000 mgr.y (mgr.25024) 207 : cluster [DBG] pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:19.678 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:17.888254+0000 mgr.y (mgr.25024) 207 : cluster [DBG] pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:20.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:12:20.573 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:19.350180+0000 mgr.y (mgr.25024) 208 : audit [DBG] from='client.25657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:20.573 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:20 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:19.350180+0000 mgr.y (mgr.25024) 208 : audit [DBG] from='client.25657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:21.646 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:19.888936+0000 mgr.y (mgr.25024) 209 : cluster [DBG] pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:21.646 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:21 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:19.888936+0000 mgr.y (mgr.25024) 209 : cluster [DBG] pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (111s) 106s ago 23m 15.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (96s) 23s ago 23m 82.1M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 23s ago 22m - - 24ecd6d5f14c f0e030541d17 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (2m) 23s ago 27m 437M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (12m) 106s ago 28m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (28m) 106s ago 29m 93.9M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (30s) 23s ago 27m 17.5M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (27m) 106s ago 27m 65.2M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:12:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 106s ago 24m 7922k - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (2m) 23s ago 24m 17.2M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (27m) 106s ago 27m 49.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (26m) 106s ago 26m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (26m) 106s ago 26m 46.2M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (26m) 106s ago 26m 50.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (25m) 23s ago 25m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (25m) 23s ago 25m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (25m) 23s ago 25m 48.6M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (25m) 23s ago 25m 52.8M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (2m) 23s ago 24m 49.3M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (23m) 106s ago 23m 41.5M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:12:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (23m) 23s ago 23m 43.5M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:12:22.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-09-17T09:12:23.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:21.889302+0000 mgr.y (mgr.25024) 210 : cluster [DBG] pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:23.867 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:22.076284+0000 mgr.y (mgr.25024) 211 : audit [DBG] from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:23.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:21.889302+0000 mgr.y (mgr.25024) 210 : cluster [DBG] pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:23.868 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:22.076284+0000 mgr.y (mgr.25024) 211 : audit [DBG] from='client.25663 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:24.646 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:12:25.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:12:25.715 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:23.889647+0000 mgr.y (mgr.25024) 212 : cluster [DBG] pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:24.633721+0000 mon.a (mon.0) 1049 : audit [DBG] from='client.? 172.21.15.27:0/1010631329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:24.845937+0000 mon.b (mon.2) 6 : audit [DBG] from='client.44177 172.21.15.105:0/2077587753' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:25.164135+0000 mon.b (mon.2) 7 : audit [INF] from='client.44183 172.21.15.105:0/1216164915' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:23.889647+0000 mgr.y (mgr.25024) 212 : cluster [DBG] pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:24.633721+0000 mon.a (mon.0) 1049 : audit [DBG] from='client.? 172.21.15.27:0/1010631329' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:25.716 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:24.845937+0000 mon.b (mon.2) 6 : audit [DBG] from='client.44177 172.21.15.105:0/2077587753' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:25.717 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:25 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:25.164135+0000 mon.b (mon.2) 7 : audit [INF] from='client.44183 172.21.15.105:0/1216164915' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:27.396 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:12:27.396 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:12:27.396 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:12:27.396 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:12:27.397 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:12:27.397 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:12:27.397 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:12:27.397 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:12:27.397 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:12:27.700 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:25.890299+0000 mgr.y (mgr.25024) 213 : cluster [DBG] pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:27.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:27 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:25.890299+0000 mgr.y (mgr.25024) 213 : cluster [DBG] pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:28.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:12:28.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:27.395068+0000 mgr.y (mgr.25024) 214 : audit [DBG] from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:28.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:28 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:27.395068+0000 mgr.y (mgr.25024) 214 : audit [DBG] from='client.25693 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:29.845 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:27.890766+0000 mgr.y (mgr.25024) 215 : cluster [DBG] pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:29.845 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:29 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:27.890766+0000 mgr.y (mgr.25024) 215 : cluster [DBG] pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:30.093 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:12:30.094 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:12:30.094 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:12:30.683 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:30.093471+0000 mon.c (mon.1) 272 : audit [DBG] from='client.? 172.21.15.27:0/1218270024' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:12:30.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:30 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:30.093471+0000 mon.c (mon.1) 272 : audit [DBG] from='client.? 172.21.15.27:0/1218270024' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:12:30.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-09-17T09:12:31.982 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:29.891434+0000 mgr.y (mgr.25024) 216 : cluster [DBG] pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:31.982 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:31 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:29.891434+0000 mgr.y (mgr.25024) 216 : cluster [DBG] pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:33.992 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:31.891769+0000 mgr.y (mgr.25024) 217 : cluster [DBG] pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:33.993 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:32.648861+0000 mgr.y (mgr.25024) 218 : audit [DBG] from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:33.993 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:31.891769+0000 mgr.y (mgr.25024) 217 : cluster [DBG] pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:33.993 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:32.648861+0000 mgr.y (mgr.25024) 218 : audit [DBG] from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:34.327 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:32.999693+0000 mgr.y (mgr.25024) 219 : audit [DBG] from='client.15777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mon", "hosts": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:34.326738+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:34.327534+0000 mon.c (mon.1) 273 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:34.328911+0000 mon.c (mon.1) 274 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:34.329614+0000 mon.c (mon.1) 275 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:34.989 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:34.335293+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:32.999693+0000 mgr.y (mgr.25024) 219 : audit [DBG] from='client.15777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "mon", "hosts": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:34.326738+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:34.327534+0000 mon.c (mon.1) 273 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:34.328911+0000 mon.c (mon.1) 274 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:34.329614+0000 mon.c (mon.1) 275 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:12:34.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:34.335293+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:35.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:12:35.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:33.892091+0000 mgr.y (mgr.25024) 220 : cluster [DBG] pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:35.821 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:34.321860+0000 mgr.y (mgr.25024) 221 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:34.330296+0000 mgr.y (mgr.25024) 222 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:34.330730+0000 mgr.y (mgr.25024) 223 : cluster [DBG] pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:34.330899+0000 mgr.y (mgr.25024) 224 : cluster [DBG] pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:34.451986+0000 mgr.y (mgr.25024) 225 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:33.892091+0000 mgr.y (mgr.25024) 220 : cluster [DBG] pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:34.321860+0000 mgr.y (mgr.25024) 221 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:34.330296+0000 mgr.y (mgr.25024) 222 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:34.330730+0000 mgr.y (mgr.25024) 223 : cluster [DBG] pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:12:35.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:34.330899+0000 mgr.y (mgr.25024) 224 : cluster [DBG] pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:35.823 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:35 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:34.451986+0000 mgr.y (mgr.25024) 225 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.149204+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:36.149785+0000 mgr.y (mgr.25024) 226 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:36.149838+0000 mgr.y (mgr.25024) 227 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.151356+0000 mon.c (mon.1) 276 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.158390+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.162429+0000 mon.c (mon.1) 277 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:36.163016+0000 mgr.y (mgr.25024) 228 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.169764+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.170939+0000 mon.c (mon.1) 278 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:36.171677+0000 mon.c (mon.1) 279 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:36.172147+0000 mgr.y (mgr.25024) 229 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:36.331547+0000 mgr.y (mgr.25024) 230 : cluster [DBG] pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.149204+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:36.149785+0000 mgr.y (mgr.25024) 226 : cephadm [INF] Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:36.149838+0000 mgr.y (mgr.25024) 227 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:12:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.151356+0000 mon.c (mon.1) 276 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.158390+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.162429+0000 mon.c (mon.1) 277 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:36.163016+0000 mgr.y (mgr.25024) 228 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.169764+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.170939+0000 mon.c (mon.1) 278 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:36.171677+0000 mon.c (mon.1) 279 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:36.172147+0000 mgr.y (mgr.25024) 229 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-17T09:12:37.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:36.331547+0000 mgr.y (mgr.25024) 230 : cluster [DBG] pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:38.670 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.141181+0000 mon.b (mon.2) 8 : audit [DBG] from='client.44219 172.21.15.105:0/2404311079' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cluster 2024-09-17T09:12:38.331913+0000 mgr.y (mgr.25024) 231 : cluster [DBG] pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.452901+0000 mon.b (mon.2) 9 : audit [INF] from='client.44225 172.21.15.105:0/2825966627' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.656983+0000 mgr.y (mgr.25024) 232 : audit [DBG] from='client.25738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.875567+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:39.204 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.876459+0000 mon.c (mon.1) 280 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.877102+0000 mon.c (mon.1) 281 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.877687+0000 mon.c (mon.1) 282 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.141181+0000 mon.b (mon.2) 8 : audit [DBG] from='client.44219 172.21.15.105:0/2404311079' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:38.331913+0000 mgr.y (mgr.25024) 231 : cluster [DBG] pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.452901+0000 mon.b (mon.2) 9 : audit [INF] from='client.44225 172.21.15.105:0/2825966627' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.656983+0000 mgr.y (mgr.25024) 232 : audit [DBG] from='client.25738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.875567+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.25024 ' entity='mgr.y' 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.876459+0000 mon.c (mon.1) 280 : audit [INF] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.877102+0000 mon.c (mon.1) 281 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:12:39.205 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.877687+0000 mon.c (mon.1) 282 : audit [DBG] from='mgr.25024 172.21.15.27:0/298056419' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:12:39.341 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:12:39.341 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 2m ago 23m 15.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:12:39.341 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (114s) 40s ago 23m 82.1M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 40s ago 23m - - 24ecd6d5f14c f0e030541d17 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (3m) 40s ago 27m 437M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (13m) 2m ago 29m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (29m) 2m ago 29m 93.9M 2048M 16.2.0 24ecd6d5f14c ac7ffcc3912d 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (47s) 40s ago 27m 17.5M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (28m) 2m ago 28m 65.2M 2048M 16.2.0 24ecd6d5f14c fb40b92a7870 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (2m) 2m ago 24m 7922k - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (2m) 40s ago 24m 17.2M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (27m) 2m ago 27m 49.6M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (27m) 2m ago 27m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (26m) 2m ago 26m 46.2M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (26m) 2m ago 26m 50.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:12:39.342 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (26m) 40s ago 26m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (25m) 40s ago 25m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (25m) 40s ago 25m 48.6M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (25m) 40s ago 25m 52.8M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (2m) 40s ago 24m 49.3M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (23m) 2m ago 23m 41.5M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:12:39.343 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (23m) 40s ago 23m 43.5M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:12:39.743 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:12:39.743 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:12:39.743 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2, 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "mds": {}, 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:12:39.744 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 12, 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:12:39.745 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:38.869445+0000 mgr.y (mgr.25024) 233 : cephadm [INF] Upgrade: Updating mon.a 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: cephadm 2024-09-17T09:12:38.878437+0000 mgr.y (mgr.25024) 234 : cephadm [INF] Deploying daemon mon.a on smithi027 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:38.998252+0000 mgr.y (mgr.25024) 235 : audit [DBG] from='client.15810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:39.332164+0000 mgr.y (mgr.25024) 236 : audit [DBG] from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: audit 2024-09-17T09:12:39.743102+0000 mon.c (mon.1) 283 : audit [DBG] from='client.? 172.21.15.27:0/1548517078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:38.869445+0000 mgr.y (mgr.25024) 233 : cephadm [INF] Upgrade: Updating mon.a 2024-09-17T09:12:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:38.878437+0000 mgr.y (mgr.25024) 234 : cephadm [INF] Deploying daemon mon.a on smithi027 2024-09-17T09:12:40.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:38.998252+0000 mgr.y (mgr.25024) 235 : audit [DBG] from='client.15810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:40.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:39.332164+0000 mgr.y (mgr.25024) 236 : audit [DBG] from='client.15816 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:40.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:39.743102+0000 mon.c (mon.1) 283 : audit [DBG] from='client.? 172.21.15.27:0/1548517078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi027", 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/2 daemons upgraded", 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T09:12:40.079 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:12:40.080 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:12:40.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:40 smithi027 systemd[1]: Stopping Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:12:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T09:12:40.685+0000 7fb420091700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-17T09:12:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a[23609]: debug 2024-09-17T09:12:40.685+0000 7fb420091700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T09:12:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:40 smithi027 podman[67174]: 2024-09-17 09:12:40.839514448 +0000 UTC m=+0.266847346 container died ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.name=CentOS Base Image, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20201204, org.label-schema.vendor=CentOS, maintainer=Dimitri Savineau , GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, ceph=True) 2024-09-17T09:12:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 podman[67174]: 2024-09-17 09:12:41.23578501 +0000 UTC m=+0.663117917 container cleanup ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20201204, RELEASE=HEAD, maintainer=Dimitri Savineau , GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T09:12:41.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 bash[67174]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a 2024-09-17T09:12:41.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 podman[67186]: 2024-09-17 09:12:41.478818488 +0000 UTC m=+0.628026163 container remove ac7ffcc3912d4a83994086f8a4b6c44bbef886ad79600e3e788c26677002e471 (image=quay.io/ceph/ceph:v16.2.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.a, org.label-schema.vendor=CentOS, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, RELEASE=HEAD, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T09:12:42.189 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-17T09:12:42.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 systemd[1]: Stopped Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:12:42.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:41 smithi027 systemd[1]: Starting Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:12:42.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 podman[67330]: 2024-09-17 09:12:42.09237849 +0000 UTC m=+0.016292174 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:12:42.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 podman[67330]: 2024-09-17 09:12:42.205314852 +0000 UTC m=+0.129228510 container create d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:12:42.449 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 podman[67330]: 2024-09-17 09:12:42.424438559 +0000 UTC m=+0.348352235 container init d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:12:42.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 podman[67330]: 2024-09-17 09:12:42.427672238 +0000 UTC m=+0.351585897 container start d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2) 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable), process ceph-mon, pid 2 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: pidfile_write: ignore empty --pid-file 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: load: jerasure load: lrc 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: RocksDB version: 7.9.2 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Git sha 0 2024-09-17T09:12:42.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Compile date 2024-09-16 15:15:13 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: DB SUMMARY 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: DB Session ID: 2NRZM5DF7MR1A86NJQMM 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: MANIFEST file: MANIFEST-000008 size: 2901 Bytes 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000053.sst 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000051.log size: 1444481 ; 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:12:42.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.env: 0x55dc5929dbc0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.info_log: 0x55dc5a272be0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.statistics: (nil) 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.use_fsync: 0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:12:42.760 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.db_log_dir: 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.wal_dir: 2024-09-17T09:12:42.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:12:42.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:12:42.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:12:42.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.write_buffer_manager: 0x55dc5a280b40 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:12:42.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.unordered_write: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.row_cache: None 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.wal_filter: None 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.wal_compression: 0 2024-09-17T09:12:42.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T09:12:42.765 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_open_files: -1 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:12:42.766 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Compression algorithms supported: 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kZSTD supported: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:12:42.767 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.merge_operator: 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_filter: None 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dc5a2727e0) 2024-09-17T09:12:42.768 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_type: 0 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_index_type: 0 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_shortening: 1 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: checksum: 4 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: no_block_cache: 0 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache: 0x55dc5a267350 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_options: 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: capacity : 536870912 2024-09-17T09:12:42.769 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_shard_bits : 4 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: strict_capacity_limit : 0 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_cache_compressed: (nil) 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: persistent_cache: (nil) 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size: 4096 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_size_deviation: 10 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_restart_interval: 16 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: index_block_restart_interval: 1 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: metadata_block_size: 4096 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: partition_filters: 0 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: use_delta_encoding: 1 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: filter_policy: bloomfilter 2024-09-17T09:12:42.770 INFO:journalctl@ceph.mon.a.smithi027.stdout: whole_key_filtering: 1 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: verify_compression: 0 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: format_version: 5 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_index_compression: 1 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: block_align: 0 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: max_auto_readahead_size: 262144 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: prepopulate_block_cache: 0 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: initial_auto_readahead_size: 8192 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression: NoCompression 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:12:42.771 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.num_levels: 7 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:12:42.772 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:12:42.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:12:42.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:12:42.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:12:42.776 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.ttl: 2592000 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T09:12:42.777 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 53.sst 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 55, last_sequence is 26453, log_number is 51,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 51 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 293d2349-0f27-4e3f-b40a-17b9ec336e0d 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564362454784, "job": 1, "event": "recovery_started", "wal_files": [51]} 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #51 mode 2 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564362459541, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 56, "file_size": 1437591, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 26454, "largest_seqno": 26782, "table_properties": {"data_size": 1434777, "index_size": 1456, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2695, "raw_average_key_size": 22, "raw_value_size": 1431320, "raw_average_value_size": 11829, "num_data_blocks": 64, "num_entries": 121, "num_filter_entries": 121, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726564362, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "293d2349-0f27-4e3f-b40a-17b9ec336e0d", "db_session_id": "2NRZM5DF7MR1A86NJQMM", "orig_file_number": 56, "seqno_to_time_mapping": "N/A"}} 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564362459654, "job": 1, "event": "recovery_finished"} 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/version_set.cc:5047] Creating manifest 58 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000051.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dc5a292e00 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: DB pointer 0x55dc5a386000 2024-09-17T09:12:42.778 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** DB Stats ** 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:12:42.779 INFO:journalctl@ceph.mon.a.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: L0 1/0 1.37 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: L6 1/0 28.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: Sum 2/0 30.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** Compaction Stats [default] ** 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 474.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.780 INFO:journalctl@ceph.mon.a.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Cumulative compaction: 0.00 GB write, 153.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Interval compaction: 0.00 GB write, 153.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache BinnedLRUCache@0x55dc5a267350#2 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.58 KB,0.000301003%) Misc(1,0.00 KB,0%) 2024-09-17T09:12:42.781 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: starting mon.a rank 0 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???) e3 preinit fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).mds e1 new map 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).mds e1 print_map 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: e1 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: legacy client fscid: -1 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout: No filesystems configured 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:12:42.782 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:12:42.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:12:42.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 ceph-mon[67349]: mon.a@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-17T09:12:42.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 bash[67330]: d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 2024-09-17T09:12:42.783 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:12:42 smithi027 systemd[1]: Started Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:12:44.254 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: ignoring --setuser ceph since I am not root 2024-09-17T09:12:44.254 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:43 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: ignoring --setgroup ceph since I am not root 2024-09-17T09:12:44.254 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:43 smithi105 ceph-mgr[60017]: -- 172.21.15.105:0/149309550 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a9962b4d00 con 0x55a996289000 2024-09-17T09:12:44.254 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.050+0000 7fa7008a9040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:12:44.254 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.106+0000 7fa7008a9040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:12:44.255 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.252+0000 7fa7008a9040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ignoring --setuser ceph since I am not root 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ignoring --setgroup ceph since I am not root 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-mgr[52513]: -- 172.21.15.27:0/3792779622 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c4499b6d00 con 0x55c44998a800 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.060+0000 7f5b2db52040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.118+0000 7f5b2db52040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:40.078780+0000 mgr.y (mgr.25024) 237 : audit [DBG] from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:40.332582+0000 mgr.y (mgr.25024) 238 : cluster [DBG] pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:42.332921+0000 mgr.y (mgr.25024) 239 : cluster [DBG] pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.874935+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-17T09:12:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.877104+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.882870+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.883525+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.883565+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.884761+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e36: y(active, since 3m), standbys: x 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.885338+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.885360+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.885373+0000 mon.a (mon.0) 9 : cluster [WRN] daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:43.896490+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e37: y(active, since 3m), standbys: x 2024-09-17T09:12:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:43.896609+0000 mon.a (mon.0) 11 : audit [INF] from='mgr.25024 ' entity='' 2024-09-17T09:12:44.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.317+0000 7fa7008a9040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:12:44.528 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.460+0000 7fa7008a9040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:12:44.639 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.274+0000 7f5b2db52040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-17T09:12:44.640 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.342+0000 7f5b2db52040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-17T09:12:44.640 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.492+0000 7f5b2db52040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:12:44.640 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.638+0000 7f5b2db52040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:12:44.906 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.599+0000 7fa7008a9040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-17T09:12:45.218 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:44.962+0000 7f5b2db52040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:12:45.218 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:45.025+0000 7f5b2db52040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:12:45.218 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:45.216+0000 7f5b2db52040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:12:45.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.905+0000 7fa7008a9040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-17T09:12:45.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:44.966+0000 7fa7008a9040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-17T09:12:45.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:45.149+0000 7fa7008a9040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-17T09:12:46.200 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:44.900572+0000 mon.a (mon.0) 12 : cluster [DBG] mgrmap e38: y(active, since 3m), standbys: x 2024-09-17T09:12:46.200 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:46.129+0000 7f5b2db52040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:12:46.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:46.015+0000 7fa7008a9040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-17T09:12:46.278 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:46.080+0000 7fa7008a9040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:12:46.507 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:46.198+0000 7f5b2db52040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-17T09:12:47.040 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:46.783+0000 7fa7008a9040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:12:47.040 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:46.899+0000 7fa7008a9040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:12:47.215 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:46.945+0000 7f5b2db52040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-17T09:12:47.215 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.065+0000 7f5b2db52040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-17T09:12:47.316 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.039+0000 7fa7008a9040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:12:47.316 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.099+0000 7fa7008a9040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:12:47.316 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.247+0000 7fa7008a9040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:12:47.316 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.314+0000 7fa7008a9040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:12:47.508 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.214+0000 7f5b2db52040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-17T09:12:47.508 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.277+0000 7f5b2db52040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-17T09:12:47.508 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.430+0000 7f5b2db52040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-17T09:12:47.599 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.374+0000 7fa7008a9040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:12:47.599 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.533+0000 7fa7008a9040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:12:47.599 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.598+0000 7fa7008a9040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:12:47.802 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.507+0000 7f5b2db52040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-17T09:12:47.802 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.570+0000 7f5b2db52040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-17T09:12:47.802 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.733+0000 7f5b2db52040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-17T09:12:47.802 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.800+0000 7f5b2db52040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-17T09:12:47.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.663+0000 7fa7008a9040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:12:47.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.743+0000 7fa7008a9040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:12:47.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.801+0000 7fa7008a9040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:12:47.861 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:47.860+0000 7fa7008a9040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:12:48.079 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.869+0000 7f5b2db52040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-17T09:12:48.080 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:47.955+0000 7f5b2db52040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-17T09:12:48.080 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:48.017+0000 7f5b2db52040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-17T09:12:48.080 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:48.078+0000 7f5b2db52040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:48.894+0000 7f5b2db52040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.642323+0000 mon.a (mon.0) 13 : cluster [DBG] Standby manager daemon x restarted 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.642441+0000 mon.a (mon.0) 14 : cluster [DBG] Standby manager daemon x started 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.643068+0000 mon.b (mon.2) 10 : audit [DBG] from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.643887+0000 mon.b (mon.2) 11 : audit [DBG] from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.644623+0000 mon.b (mon.2) 12 : audit [DBG] from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:12:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:48 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.644890+0000 mon.b (mon.2) 13 : audit [DBG] from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:12:49.028 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:12:48 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x[60013]: 2024-09-17T09:12:48.639+0000 7fa7008a9040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-17T09:12:50.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:49.912+0000 7f5acd221640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.698829+0000 mon.a (mon.0) 15 : cluster [DBG] mgrmap e39: y(active, since 3m), standbys: x 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.896815+0000 mon.a (mon.0) 16 : cluster [INF] Active manager daemon y restarted 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.897121+0000 mon.a (mon.0) 17 : cluster [INF] Activating manager daemon y 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.902034+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.902588+0000 mon.a (mon.0) 19 : cluster [DBG] mgrmap e40: y(active, starting, since 0.00554611s), standbys: x 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.904718+0000 mon.a (mon.0) 20 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.904925+0000 mon.a (mon.0) 21 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:12:50.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.905101+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.906756+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.906922+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907061+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907241+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907389+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907571+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907721+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.907877+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.908035+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.908295+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.908503+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.908628+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.908993+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:12:50.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:48.915745+0000 mon.a (mon.0) 36 : cluster [INF] Manager daemon y is now available 2024-09-17T09:12:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.933651+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:12:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.959726+0000 mon.a (mon.0) 38 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:12:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:49 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:48.978163+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:49.906167+0000 mon.a (mon.0) 40 : cluster [DBG] mgrmap e41: y(active, since 1.00912s), standbys: x 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:49.917572+0000 mgr.y (mgr.54108) 1 : cluster [DBG] pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:50.823+0000 7f5a9391c640 -1 log_channel(cephadm) log [ERR] : [17/Sep/2024:09:12:50] ENGINE Error in HTTPServer.serve 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: Traceback (most recent call last): 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._connections.run(self.expiration_interval) 2024-09-17T09:12:51.258 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._run(expiration_interval) 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: s = self.context.wrap_socket( 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:12:51.259 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: return self.sslsocket_class._create( 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self.do_handshake() 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: self._sslobj.do_handshake() 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:12:51.260 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:12:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:12:52.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:50.700481+0000 mgr.y (mgr.54108) 2 : cephadm [INF] [17/Sep/2024:09:12:50] ENGINE Bus STARTING 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:50.821892+0000 mgr.y (mgr.54108) 3 : cephadm [INF] [17/Sep/2024:09:12:50] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:50.824508+0000 mgr.y (mgr.54108) 4 : cephadm [ERR] [17/Sep/2024:09:12:50] ENGINE Error in HTTPServer.serve 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: Traceback (most recent call last): 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._connections.run(self.expiration_interval) 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._run(expiration_interval) 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:12:52.246 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: s = self.context.wrap_socket( 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: return self.sslsocket_class._create( 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self.do_handshake() 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: self._sslobj.do_handshake() 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:50.902037+0000 mon.a (mon.0) 41 : cluster [INF] Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:50.902056+0000 mon.a (mon.0) 42 : cluster [INF] Cluster is now healthy 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:50.908258+0000 mgr.y (mgr.54108) 5 : cluster [DBG] pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:50.913645+0000 mon.a (mon.0) 43 : cluster [DBG] mgrmap e42: y(active, since 2s), standbys: x 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:50.922905+0000 mgr.y (mgr.54108) 6 : cephadm [INF] [17/Sep/2024:09:12:50] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:12:52.247 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:51 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:50.922989+0000 mgr.y (mgr.54108) 7 : cephadm [INF] [17/Sep/2024:09:12:50] ENGINE Bus STARTED 2024-09-17T09:12:53.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:51.920438+0000 mon.b (mon.2) 14 : audit [DBG] from='client.44306 172.21.15.105:0/1324074898' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:12:53.249 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:52 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:52.303351+0000 mon.b (mon.2) 15 : audit [INF] from='client.44312 172.21.15.105:0/1770798426' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:12:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:52.908585+0000 mgr.y (mgr.54108) 8 : cluster [DBG] pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:52.930045+0000 mon.a (mon.0) 44 : cluster [DBG] mgrmap e43: y(active, since 4s), standbys: x 2024-09-17T09:12:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:53.649318+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:53 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:53.654186+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:55.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:54.199769+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:55.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:54.207029+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:55.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:54.908898+0000 mgr.y (mgr.54108) 9 : cluster [DBG] pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:12:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:56.316859+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:56.323367+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:56.846941+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:56.850832+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:57.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:56.909501+0000 mgr.y (mgr.54108) 10 : cluster [DBG] pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T09:12:57.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:57.285006+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:57.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:57 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:57.289931+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:12:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:58 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:57.280992+0000 mgr.y (mgr.54108) 11 : cephadm [INF] Detected new or changed devices on smithi027 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:58.909908+0000 mgr.y (mgr.54108) 12 : cluster [DBG] pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.634465+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.639817+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.641403+0000 mon.a (mon.0) 57 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.642258+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.647644+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.768857+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.770972+0000 mon.a (mon.0) 61 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:00.231 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.771944+0000 mon.a (mon.0) 62 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:13:00.232 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:12:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:12:59.772543+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T09:13:01.112 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:59.642984+0000 mgr.y (mgr.54108) 13 : cephadm [WRN] unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:01.112 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:59.643632+0000 mgr.y (mgr.54108) 14 : cluster [DBG] pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:13:01.112 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:12:59.772983+0000 mgr.y (mgr.54108) 15 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-17T09:13:01.112 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:00 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:12:59.852509+0000 mon.a (mon.0) 64 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:13:02.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:13:01.180394+0000 mgr.y (mgr.54108) 16 : cephadm [INF] Upgrade: Updating mon.c 2024-09-17T09:13:02.447 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:13:01.186038+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:02.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:13:01.186834+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:02.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:13:01.187431+0000 mon.a (mon.0) 67 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:02.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: audit 2024-09-17T09:13:01.188015+0000 mon.a (mon.0) 68 : audit [DBG] from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:02.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cephadm 2024-09-17T09:13:01.188771+0000 mgr.y (mgr.54108) 17 : cephadm [INF] Deploying daemon mon.c on smithi027 2024-09-17T09:13:02.448 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: cluster 2024-09-17T09:13:01.644332+0000 mgr.y (mgr.54108) 18 : cluster [DBG] pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:03.102 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 systemd[1]: Stopping Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:13:03.103 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: debug 2024-09-17T09:13:02.901+0000 7f561311d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-17T09:13:03.103 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:02 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c[26924]: debug 2024-09-17T09:13:02.901+0000 7f561311d700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T09:13:03.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:03 smithi027 podman[71564]: 2024-09-17 09:13:03.106130247 +0000 UTC m=+0.305502180 container died fb40b92a7870dd735dbad0c6e863a4c25443f59d443bcb756b126f85816d993a (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c, RELEASE=HEAD, org.label-schema.build-date=20201204, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Dimitri Savineau , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image) 2024-09-17T09:13:03.744 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:03 smithi027 podman[71564]: 2024-09-17 09:13:03.459731669 +0000 UTC m=+0.659103592 container cleanup fb40b92a7870dd735dbad0c6e863a4c25443f59d443bcb756b126f85816d993a (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c, RELEASE=HEAD, CEPH_POINT_RELEASE=-16.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Dimitri Savineau , org.label-schema.name=CentOS Base Image, ceph=True, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T09:13:03.744 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:03 smithi027 bash[71564]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c 2024-09-17T09:13:04.007 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:03 smithi027 podman[71577]: 2024-09-17 09:13:03.744381411 +0000 UTC m=+0.634774818 container remove fb40b92a7870dd735dbad0c6e863a4c25443f59d443bcb756b126f85816d993a (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon.c, org.label-schema.license=GPLv2, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Dimitri Savineau , GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, RELEASE=HEAD, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, ceph=True, org.label-schema.build-date=20201204, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T09:13:04.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-17T09:13:04.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 systemd[1]: Stopped Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:13:04.429 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 systemd[1]: Starting Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:13:04.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 podman[71719]: 2024-09-17 09:13:04.428978726 +0000 UTC m=+0.108308342 container create 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux ) 2024-09-17T09:13:04.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 podman[71719]: 2024-09-17 09:13:04.336946663 +0000 UTC m=+0.016276236 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 podman[71719]: 2024-09-17 09:13:04.639746152 +0000 UTC m=+0.319075714 container init 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 podman[71719]: 2024-09-17 09:13:04.642975167 +0000 UTC m=+0.322304729 container start 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0) 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable), process ceph-mon, pid 2 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: pidfile_write: ignore empty --pid-file 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: load: jerasure load: lrc 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: RocksDB version: 7.9.2 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Git sha 0 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Compile date 2024-09-16 15:15:13 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: DB SUMMARY 2024-09-17T09:13:04.682 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: DB Session ID: LV04JQB6C251PMUR8V7C 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: MANIFEST file: MANIFEST-000005 size: 3023 Bytes 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000053.sst 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000051.log size: 3549277 ; 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.env: 0x55f332318bc0 2024-09-17T09:13:04.683 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.info_log: 0x55f332ba0be0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.statistics: (nil) 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.use_fsync: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:13:04.684 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.db_log_dir: 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.wal_dir: 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:13:04.685 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.write_buffer_manager: 0x55f332baeb40 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:13:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:13:04.687 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.unordered_write: 0 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.row_cache: None 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.wal_filter: None 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:13:04.688 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.wal_compression: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:13:04.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:13:04.690 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_open_files: -1 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Compression algorithms supported: 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kZSTD supported: 0 2024-09-17T09:13:04.691 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.merge_operator: 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_filter: None 2024-09-17T09:13:04.692 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f332ba07e0) 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_type: 0 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_index_type: 0 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_shortening: 1 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:13:04.693 INFO:journalctl@ceph.mon.c.smithi027.stdout: checksum: 4 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: no_block_cache: 0 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache: 0x55f332b95350 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_options: 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: capacity : 536870912 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_shard_bits : 4 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: strict_capacity_limit : 0 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_cache_compressed: (nil) 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: persistent_cache: (nil) 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size: 4096 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_size_deviation: 10 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_restart_interval: 16 2024-09-17T09:13:04.694 INFO:journalctl@ceph.mon.c.smithi027.stdout: index_block_restart_interval: 1 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: metadata_block_size: 4096 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: partition_filters: 0 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: use_delta_encoding: 1 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: filter_policy: bloomfilter 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: whole_key_filtering: 1 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: verify_compression: 0 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: format_version: 5 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_index_compression: 1 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: block_align: 0 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: max_auto_readahead_size: 262144 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: prepopulate_block_cache: 0 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: initial_auto_readahead_size: 8192 2024-09-17T09:13:04.695 INFO:journalctl@ceph.mon.c.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression: NoCompression 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.num_levels: 7 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:13:04.696 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:13:04.697 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:13:04.698 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:13:04.699 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:13:04.700 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:13:04.701 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.ttl: 2592000 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T09:13:04.702 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 53.sst 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 55, last_sequence is 29433, log_number is 51,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 51 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dde0bb70-0365-47dd-a3ac-0ad4c6737b5b 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564384670633, "job": 1, "event": "recovery_started", "wal_files": [51]} 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #51 mode 2 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564384680778, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 56, "file_size": 2314586, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 29438, "largest_seqno": 29759, "table_properties": {"data_size": 2311601, "index_size": 1623, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2496, "raw_average_key_size": 21, "raw_value_size": 2308028, "raw_average_value_size": 19896, "num_data_blocks": 72, "num_entries": 116, "num_filter_entries": 116, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726564384, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dde0bb70-0365-47dd-a3ac-0ad4c6737b5b", "db_session_id": "LV04JQB6C251PMUR8V7C", "orig_file_number": 56, "seqno_to_time_mapping": "N/A"}} 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726564384680906, "job": 1, "event": "recovery_finished"} 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/version_set.cc:5047] Creating manifest 58 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000051.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f332bc0e00 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: DB pointer 0x55f332cb4000 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T09:13:04.703 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** DB Stats ** 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T09:13:04.704 INFO:journalctl@ceph.mon.c.smithi027.stdout: L0 1/0 2.21 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 674.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: L6 1/0 30.54 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Sum 2/0 32.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 674.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 674.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** Compaction Stats [default] ** 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:13:04.705 INFO:journalctl@ceph.mon.c.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 674.2 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Cumulative compaction: 0.00 GB write, 165.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Interval compaction: 0.00 GB write, 165.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache BinnedLRUCache@0x55f332b95350#2 capacity: 512.00 MB usage: 2.20 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-09-17T09:13:04.706 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:04.707 INFO:journalctl@ceph.mon.c.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:13:04.707 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: starting mon.c rank 1 at public addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] at bind addrs [v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???) e3 preinit fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).mds e1 new map 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).mds e1 print_map 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout: e1 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:13:05.008 INFO:journalctl@ceph.mon.c.smithi027.stdout: legacy client fscid: -1 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout: 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout: No filesystems configured 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 ceph-mon[71738]: mon.c@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 bash[71719]: 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c 2024-09-17T09:13:05.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:04 smithi027 systemd[1]: Started Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:13:06.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mkfs d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: It appears safe to stop mon.b 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Updating mon.b 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Deploying daemon mon.b on smithi105 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Detected new or changed devices on smithi105 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:06.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:13:06.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: It appears safe to stop mon.a 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Updating mon.a 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Deploying daemon mon.a on smithi027 2024-09-17T09:13:06.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.a calling monitor election 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: fsmap 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mkfs d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:13:06.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: It appears safe to stop mon.b 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Updating mon.b 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Deploying daemon mon.b on smithi105 2024-09-17T09:13:06.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Detected new or changed devices on smithi105 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:06.517 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.518 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.25024 ' entity='' 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e38: y(active, since 3m), standbys: x 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Standby manager daemon x restarted 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Standby manager daemon x started 2024-09-17T09:13:06.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e39: y(active, since 3m), standbys: x 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Active manager daemon y restarted 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Activating manager daemon y 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e40: y(active, starting, since 0.00554611s), standbys: x 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:13:06.520 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.522 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: It appears safe to stop mon.a 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Updating mon.a 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Deploying daemon mon.a on smithi027 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.523 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.a calling monitor election 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: fsmap 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Manager daemon y is now available 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.524 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e41: y(active, since 1.00912s), standbys: x 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [17/Sep/2024:09:12:50] ENGINE Bus STARTING 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [17/Sep/2024:09:12:50] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [17/Sep/2024:09:12:50] ENGINE Error in HTTPServer.serve 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: Traceback (most recent call last): 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._run(expiration_interval) 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:13:06.525 INFO:journalctl@ceph.mon.c.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: s = self.context.wrap_socket( 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: return self.sslsocket_class._create( 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: self.do_handshake() 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: self._sslobj.do_handshake() 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Cluster is now healthy 2024-09-17T09:13:06.526 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e42: y(active, since 2s), standbys: x 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [17/Sep/2024:09:12:50] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [17/Sep/2024:09:12:50] ENGINE Bus STARTED 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='client.44306 172.21.15.105:0/1324074898' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.527 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.25024 ' entity='' 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e38: y(active, since 3m), standbys: x 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Standby manager daemon x restarted 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Standby manager daemon x started 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:13:06.528 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e39: y(active, since 3m), standbys: x 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Active manager daemon y restarted 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Activating manager daemon y 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e40: y(active, starting, since 0.00554611s), standbys: x 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:13:06.529 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='client.44312 172.21.15.105:0/1770798426' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e43: y(active, since 4s), standbys: x 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.530 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Detected new or changed devices on smithi027 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.531 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: It appears safe to stop mon.c 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Upgrade: Updating mon.c 2024-09-17T09:13:06.532 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Manager daemon y is now available 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:13:06.533 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e41: y(active, since 1.00912s), standbys: x 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [17/Sep/2024:09:12:50] ENGINE Bus STARTING 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [17/Sep/2024:09:12:50] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [17/Sep/2024:09:12:50] ENGINE Error in HTTPServer.serve 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: Traceback (most recent call last): 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._run(expiration_interval) 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:13:06.534 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: s = self.context.wrap_socket( 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: return self.sslsocket_class._create( 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: self.do_handshake() 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: self._sslobj.do_handshake() 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Cluster is now healthy 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e42: y(active, since 2s), standbys: x 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [17/Sep/2024:09:12:50] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:13:06.535 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [17/Sep/2024:09:12:50] ENGINE Bus STARTED 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='client.44306 172.21.15.105:0/1324074898' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Deploying daemon mon.c on smithi027 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.c calling monitor election 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.a calling monitor election 2024-09-17T09:13:06.536 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.b calling monitor election 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: monmap e4: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: fsmap 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: mgrmap e43: y(active, since 16s), standbys: x 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.537 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[71738]: pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='client.44312 172.21.15.105:0/1770798426' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e43: y(active, since 4s), standbys: x 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.538 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Detected new or changed devices on smithi027 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.539 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: It appears safe to stop mon.c 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Upgrade: Updating mon.c 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:06.540 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Deploying daemon mon.c on smithi027 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.c calling monitor election 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.a calling monitor election 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.b calling monitor election 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: monmap e4: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: fsmap 2024-09-17T09:13:06.541 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: mgrmap e43: y(active, since 16s), standbys: x 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.542 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:06 smithi027 ceph-mon[67349]: pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:06.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mkfs d61dd7c6-74d0-11ef-bceb-c7b262605968 2024-09-17T09:13:06.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:13:06.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: It appears safe to stop mon.b 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Updating mon.b 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Deploying daemon mon.b on smithi105 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Detected new or changed devices on smithi105 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:06.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:13:06.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v113: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v114: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v115: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v116: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v117: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v118: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v119: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v120: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 98 B/s rd, 0 op/s 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v121: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: It appears safe to stop mon.a 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v122: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v123: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Updating mon.a 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Deploying daemon mon.a on smithi027 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='client.25762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v124: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v125: 161 pgs: 161 active+clean; 604 KiB data, 93 MiB used, 715 GiB / 715 GiB avail; 121 B/s rd, 0 op/s 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.a calling monitor election 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: monmap e3: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: fsmap 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.25024 ' entity='' 2024-09-17T09:13:06.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e38: y(active, since 3m), standbys: x 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Standby manager daemon x restarted 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Standby manager daemon x started 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.? 172.21.15.105:0/3938694451' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e39: y(active, since 3m), standbys: x 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Active manager daemon y restarted 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Activating manager daemon y 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e40: y(active, starting, since 0.00554611s), standbys: x 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:13:06.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Manager daemon y is now available 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e41: y(active, since 1.00912s), standbys: x 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v3: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [17/Sep/2024:09:12:50] ENGINE Bus STARTING 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [17/Sep/2024:09:12:50] ENGINE Serving on https://172.21.15.27:7150 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [17/Sep/2024:09:12:50] ENGINE Error in HTTPServer.serve 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-17T09:13:06.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:13:06.552 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Cluster is now healthy 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v4: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e42: y(active, since 2s), standbys: x 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [17/Sep/2024:09:12:50] ENGINE Serving on http://172.21.15.27:8765 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [17/Sep/2024:09:12:50] ENGINE Bus STARTED 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='client.44306 172.21.15.105:0/1324074898' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist ls"}]: dispatch 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='client.44312 172.21.15.105:0/1770798426' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blacklist", "blacklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: access denied 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v5: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e43: y(active, since 4s), standbys: x 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v6: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v7: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Detected new or changed devices on smithi027 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v8: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v9: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 12 op/s 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: It appears safe to stop mon.c 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Upgrade: Updating mon.c 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:06.555 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Deploying daemon mon.c on smithi027 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v10: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.c calling monitor election 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.a calling monitor election 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.b calling monitor election 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: monmap e4: 3 mons at {a=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.27:3301/0,v1:172.21.15.27:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: fsmap 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:13:06.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: mgrmap e43: y(active, since 16s), standbys: x 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:06.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:06 smithi105 ceph-mon[70467]: pgmap v12: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-17T09:13:09.007 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:08 smithi027 ceph-mon[67349]: pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:08 smithi027 ceph-mon[71738]: pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:09.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:08 smithi105 ceph-mon[70467]: pgmap v13: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:09.987 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:13:09.691+0000 7f5acd221640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[71738]: pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:09 smithi027 ceph-mon[67349]: pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:10.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:09 smithi105 ceph-mon[70467]: pgmap v14: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:10.437 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (2m) 2s ago 24m 26.9M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (2m) 14s ago 24m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 14s ago 23m - - 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (3m) 14s ago 28m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (13m) 2s ago 29m 470M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (29s) 2s ago 29m 40.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (79s) 14s ago 28m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6s) 2s ago 28m 20.8M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 2s ago 24m 16.5M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:13:11.111 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (3m) 14s ago 24m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (27m) 2s ago 27m 49.0M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (27m) 2s ago 27m 51.1M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (27m) 2s ago 27m 46.6M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (27m) 2s ago 27m 50.4M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (26m) 14s ago 26m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (26m) 14s ago 26m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (26m) 14s ago 26m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (25m) 14s ago 25m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (2m) 14s ago 25m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (23m) 2s ago 23m 42.7M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:13:11.112 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (23m) 14s ago 23m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:13:11.523 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:13:11.523 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:13:11.523 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:13:11.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:13:11.523 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 10, 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 5 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:13:11.524 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:13:11.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:11 smithi027 ceph-mon[71738]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.774 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:11 smithi027 ceph-mon[71738]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:11 smithi027 ceph-mon[67349]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.774 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:11 smithi027 ceph-mon[67349]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:11 smithi105 ceph-mon[70467]: from='client.64102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:11 smithi105 ceph-mon[70467]: from='client.54168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:11.872 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi027", 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "mon" 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/2 daemons upgraded", 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:13:11.873 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='client.54180 172.21.15.27:0/3015423134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='client.54180 172.21.15.27:0/3015423134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:12.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:12 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='client.64114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='client.54180 172.21.15.27:0/3015423134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: pgmap v15: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='client.64126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:12 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: Reconfiguring mon.a (monmap changed)... 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[67349]: Reconfiguring daemon mon.a on smithi027 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: Reconfiguring mon.a (monmap changed)... 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:13.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:13 smithi027 ceph-mon[71738]: Reconfiguring daemon mon.a on smithi027 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: pgmap v16: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: pgmap v17: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: Reconfiguring mon.a (monmap changed)... 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:13.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:13.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:13 smithi105 ceph-mon[70467]: Reconfiguring daemon mon.a on smithi027 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[67349]: Reconfiguring daemon mgr.y on smithi027 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.097 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T09:13:15.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:15.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:15.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:15.098 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:14 smithi027 ceph-mon[71738]: Reconfiguring daemon mgr.y on smithi027 2024-09-17T09:13:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: Reconfiguring mgr.y (monmap changed)... 2024-09-17T09:13:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:15.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:14 smithi105 ceph-mon[70467]: Reconfiguring daemon mgr.y on smithi027 2024-09-17T09:13:16.234 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.235 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:16.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:16.236 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: pgmap v18: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: Reconfiguring mon.c (monmap changed)... 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: Reconfiguring daemon mon.c on smithi027 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:17.122 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: Reconfiguring mon.c (monmap changed)... 2024-09-17T09:13:17.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: Reconfiguring daemon mon.c on smithi027 2024-09-17T09:13:17.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:13:17.123 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:16 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:17.211 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: Reconfiguring mon.c (monmap changed)... 2024-09-17T09:13:17.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: Reconfiguring daemon mon.c on smithi027 2024-09-17T09:13:17.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:17.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:13:17.212 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:16 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[67349]: pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[67349]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T09:13:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.0 on smithi027 2024-09-17T09:13:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[71738]: pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[71738]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T09:13:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:17 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.0 on smithi027 2024-09-17T09:13:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:17 smithi105 ceph-mon[70467]: pgmap v19: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:17 smithi105 ceph-mon[70467]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T09:13:18.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:17 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.0 on smithi027 2024-09-17T09:13:19.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.241 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.1 on smithi027 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:19.242 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:18 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.1 on smithi027 2024-09-17T09:13:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T09:13:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:13:19.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:19.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:18 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.1 on smithi027 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[67349]: pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[71738]: pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:20.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:13:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:19 smithi105 ceph-mon[70467]: pgmap v20: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:20.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:13:20.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.2 on smithi027 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:21.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T09:13:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.2 on smithi027 2024-09-17T09:13:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:13:21.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T09:13:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.2 on smithi027 2024-09-17T09:13:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:21.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:13:21.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:22.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[67349]: pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[67349]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T09:13:22.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.3 on smithi027 2024-09-17T09:13:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[71738]: pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[71738]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T09:13:22.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:21 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.3 on smithi027 2024-09-17T09:13:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:21 smithi105 ceph-mon[70467]: pgmap v21: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:21 smithi105 ceph-mon[70467]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T09:13:22.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:21 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.3 on smithi027 2024-09-17T09:13:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:23.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:23.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: Reconfiguring rgw.foo.smithi027.yrcgts (monmap changed)... 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: Reconfiguring daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:24.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: Reconfiguring rgw.foo.smithi027.yrcgts (monmap changed)... 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: Reconfiguring daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:24.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: Reconfiguring rgw.foo.smithi027.yrcgts (monmap changed)... 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: Reconfiguring daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: pgmap v22: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:13:24.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:13:24.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:25.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: Reconfiguring mon.b (monmap changed)... 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: Reconfiguring daemon mon.b on smithi105 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:25.223 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: Reconfiguring mon.b (monmap changed)... 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: Reconfiguring daemon mon.b on smithi105 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:25.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: Reconfiguring mon.b (monmap changed)... 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: Reconfiguring daemon mon.b on smithi105 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:13:25.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:26.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:26.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T09:13:26.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T09:13:26.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.207 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T09:13:26.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:13:26.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:26.208 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:25 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:26.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: pgmap v23: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: Reconfiguring mgr.x (monmap changed)... 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: Reconfiguring daemon mgr.x on smithi105 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:26.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:25 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.4 on smithi105 2024-09-17T09:13:27.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:26 smithi027 ceph-mon[67349]: pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:26 smithi027 ceph-mon[71738]: pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:26 smithi105 ceph-mon[70467]: pgmap v24: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:28.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:28.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:28 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:28 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.5 on smithi105 2024-09-17T09:13:29.881 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:29.881 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:29.881 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:29.881 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T09:13:29.881 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:13:29.882 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:29.882 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:29 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T09:13:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T09:13:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: pgmap v25: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:30.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T09:13:30.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:13:30.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:30.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:29 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.6 on smithi105 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:13:31.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:13:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T09:13:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T09:13:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:32.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:32.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:32.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T09:13:32.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T09:13:32.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:32.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:32.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: Reconfiguring daemon osd.7 on smithi105 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: pgmap v26: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:13:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: Reconfiguring rgw.foo.smithi105.afiseg (monmap changed)... 2024-09-17T09:13:33.387 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: Reconfiguring daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T09:13:33.388 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.390 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.391 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: Reconfiguring rgw.foo.smithi105.afiseg (monmap changed)... 2024-09-17T09:13:33.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: Reconfiguring daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:13:33.392 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.393 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T09:13:33.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T09:13:33.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.394 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.395 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.397 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:13:33.398 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.399 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.400 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:13:33.401 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:13:33.402 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:13:33.403 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:13:33.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:13:33.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:13:33.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:13:33.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:13:33.404 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.405 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:13:33.406 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.408 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:32 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: Reconfiguring rgw.foo.smithi105.afiseg (monmap changed)... 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: Reconfiguring daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.410 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.411 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.412 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.413 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.413 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.413 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.413 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:13:33.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:13:33.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:13:33.416 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:13:33.417 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:32 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:13:34.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:34.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:34.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:34.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.262 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: pgmap v27: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mds 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nfs 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all node-exporter 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all prometheus 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all alertmanager 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all grafana 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all loki 2024-09-17T09:13:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all promtail 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: pgmap v28: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: pgmap v29: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: pgmap v30: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: pgmap v31: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:13:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:34 smithi027 ceph-mon[67349]: pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:34 smithi027 ceph-mon[71738]: pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:34 smithi105 ceph-mon[70467]: pgmap v32: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:38.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:37 smithi027 ceph-mon[67349]: pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:38.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:37 smithi027 ceph-mon[71738]: pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:38.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:37 smithi105 ceph-mon[70467]: pgmap v33: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:39 smithi027 ceph-mon[67349]: pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:39 smithi027 ceph-mon[71738]: pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:39 smithi105 ceph-mon[70467]: pgmap v34: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:42.240 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:41 smithi027 ceph-mon[71738]: pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:42.241 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:41 smithi027 ceph-mon[67349]: pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:41 smithi105 ceph-mon[70467]: pgmap v35: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:42.802 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:13:44.115 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:43 smithi027 ceph-mon[67349]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:44.116 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:43 smithi027 ceph-mon[67349]: pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:44.116 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:43 smithi027 ceph-mon[71738]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:44.116 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:43 smithi027 ceph-mon[71738]: pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:43 smithi105 ceph-mon[70467]: from='client.54192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:43 smithi105 ceph-mon[70467]: pgmap v36: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:44.808 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 35s ago 24m 26.9M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (2m) 48s ago 24m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 48s ago 24m - - 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (4m) 48s ago 28m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (14m) 35s ago 30m 470M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (62s) 35s ago 30m 40.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (113s) 48s ago 29m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (40s) 35s ago 29m 20.8M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (3m) 35s ago 25m 16.5M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (3m) 48s ago 25m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (28m) 35s ago 28m 49.0M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (28m) 35s ago 28m 51.1M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:13:44.809 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (27m) 35s ago 27m 46.6M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (27m) 35s ago 27m 50.4M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (27m) 48s ago 27m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (27m) 48s ago 27m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (26m) 48s ago 26m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (26m) 48s ago 26m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (3m) 48s ago 25m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (24m) 35s ago 24m 42.7M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:13:44.810 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (24m) 48s ago 24m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:13:45.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-09-17T09:13:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:45 smithi027 ceph-mon[71738]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:45 smithi027 ceph-mon[71738]: pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:45 smithi027 ceph-mon[67349]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:45 smithi027 ceph-mon[67349]: pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:45 smithi105 ceph-mon[70467]: from='client.54198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:45 smithi105 ceph-mon[70467]: pgmap v37: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:47.030 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:47 smithi027 ceph-mon[67349]: pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:47.030 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:47 smithi027 ceph-mon[71738]: pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:47 smithi105 ceph-mon[70467]: pgmap v38: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:47.433 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:13:48.081 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:48 smithi027 ceph-mon[71738]: from='client.54204 172.21.15.27:0/1654224022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:48.081 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:48 smithi027 ceph-mon[67349]: from='client.54204 172.21.15.27:0/1654224022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:48.106 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-09-17T09:13:48.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:48 smithi105 ceph-mon[70467]: from='client.54204 172.21.15.27:0/1654224022' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:49.389 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:49 smithi027 ceph-mon[71738]: pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:49.389 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:49 smithi027 ceph-mon[67349]: pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:49 smithi105 ceph-mon[70467]: pgmap v39: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:50.187 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)" 2024-09-17T09:13:50.462 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:50 smithi027 ceph-mon[67349]: from='client.54210 172.21.15.27:0/3977817440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:50.462 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:50 smithi027 ceph-mon[71738]: from='client.54210 172.21.15.27:0/3977817440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:50 smithi105 ceph-mon[70467]: from='client.54210 172.21.15.27:0/3977817440' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:13:50.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-09-17T09:13:51.366 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:51 smithi027 ceph-mon[67349]: pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:51.366 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:51 smithi027 ceph-mon[71738]: pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:51 smithi105 ceph-mon[70467]: pgmap v40: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:54.115 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:13:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:53 smithi027 ceph-mon[67349]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:53 smithi027 ceph-mon[67349]: pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:53 smithi027 ceph-mon[71738]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:54.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:53 smithi027 ceph-mon[71738]: pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:53 smithi105 ceph-mon[70467]: from='client.54216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:53 smithi105 ceph-mon[70467]: pgmap v41: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:54.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:13:56.200 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:55 smithi027 ceph-mon[67349]: pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:56.200 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:55 smithi027 ceph-mon[71738]: pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:55 smithi105 ceph-mon[70467]: pgmap v42: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:56.687 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:13:56.688 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:13:57.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:13:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:57 smithi027 ceph-mon[71738]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:57 smithi027 ceph-mon[71738]: pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:57 smithi027 ceph-mon[67349]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:57 smithi027 ceph-mon[67349]: pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:57 smithi105 ceph-mon[70467]: from='client.54222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:13:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:57 smithi105 ceph-mon[70467]: pgmap v43: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:13:59.565 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:13:59.565 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:13:59.565 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:14:00.255 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:59 smithi027 ceph-mon[71738]: pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:00.255 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:13:59 smithi027 ceph-mon[71738]: from='client.64168 172.21.15.27:0/286826746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:14:00.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:59 smithi027 ceph-mon[67349]: pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:00.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:13:59 smithi027 ceph-mon[67349]: from='client.64168 172.21.15.27:0/286826746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:14:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:59 smithi105 ceph-mon[70467]: pgmap v44: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:13:59 smithi105 ceph-mon[70467]: from='client.64168 172.21.15.27:0/286826746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:14:00.282 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-09-17T09:14:01.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:00 smithi027 ceph-mon[67349]: pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:01.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:00 smithi027 ceph-mon[71738]: pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:01.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:00 smithi105 ceph-mon[70467]: pgmap v45: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:03.624 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:04.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='client.64174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: pgmap v46: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:04.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:04.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:14:05.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:05.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:05.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[67349]: pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:05.252 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[67349]: pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:05.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:05.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:05.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[71738]: pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:05.253 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:04 smithi027 ceph-mon[71738]: pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:04 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:04 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:04 smithi105 ceph-mon[70467]: pgmap v47: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:05.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:04 smithi105 ceph-mon[70467]: pgmap v48: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:05 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:06.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:05 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:05 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: pgmap v49: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:07.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:07.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:07.447 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (3m) 59s ago 25m 26.9M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (3m) 71s ago 25m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 71s ago 24m - - 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (4m) 71s ago 29m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (14m) 59s ago 30m 470M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (86s) 59s ago 30m 40.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (2m) 71s ago 29m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:14:08.117 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (63s) 59s ago 29m 20.8M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 59s ago 25m 16.5M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (4m) 71s ago 25m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (28m) 59s ago 28m 49.0M 4096M 16.2.0 24ecd6d5f14c 0249074d5c8d 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (28m) 59s ago 28m 51.1M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (28m) 59s ago 28m 46.6M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (27m) 59s ago 27m 50.4M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (27m) 71s ago 27m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (27m) 71s ago 27m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (27m) 71s ago 27m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (26m) 71s ago 26m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (3m) 71s ago 26m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (24m) 59s ago 24m 42.7M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:14:08.118 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (24m) 71s ago 24m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: Upgrade: osd.0 is safe to restart 2024-09-17T09:14:08.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[67349]: pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: Upgrade: osd.0 is safe to restart 2024-09-17T09:14:08.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:07 smithi027 ceph-mon[71738]: pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:14:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:14:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:14:08.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: Upgrade: osd.0 is safe to restart 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: from='client.64180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: from='client.64186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:08.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:07 smithi105 ceph-mon[70467]: pgmap v50: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:08.536 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:14:08.536 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:14:08.536 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:14:08.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:08.536 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 10, 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 5 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:14:08.537 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/8 daemons upgraded", 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:14:08.882 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:14:09.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:09.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: Upgrade: Updating osd.0 2024-09-17T09:14:09.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:09.720 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: Deploying daemon osd.0 on smithi027 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='client.64198 172.21.15.27:0/3270178560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[67349]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: Upgrade: Updating osd.0 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:09.721 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: Deploying daemon osd.0 on smithi027 2024-09-17T09:14:09.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='client.64198 172.21.15.27:0/3270178560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:09.722 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:09 smithi027 ceph-mon[71738]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='client.64192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: Upgrade: Updating osd.0 2024-09-17T09:14:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:14:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:09.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: Deploying daemon osd.0 on smithi027 2024-09-17T09:14:09.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='client.64198 172.21.15.27:0/3270178560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:09.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:09 smithi105 ceph-mon[70467]: from='client.54264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:10 smithi027 ceph-mon[67349]: pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:10 smithi027 ceph-mon[71738]: pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:10 smithi105 ceph-mon[70467]: pgmap v51: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:11.968 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:11 smithi027 systemd[1]: Stopping Ceph osd.0 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:14:11.968 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T09:14:11.681+0000 7fb0b3b08700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:14:11.968 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T09:14:11.681+0000 7fb0b3b08700 -1 osd.0 77 *** Got signal Terminated *** 2024-09-17T09:14:11.968 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0[29231]: debug 2024-09-17T09:14:11.681+0000 7fb0b3b08700 -1 osd.0 77 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:14:11.968 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:11 smithi027 podman[78119]: 2024-09-17 09:14:11.713039186 +0000 UTC m=+0.134029344 container died 0249074d5c8d2c1c9bdd2e770b4bdd3ec71e27dde5eaf975d6d2a4859f665993 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0, ceph=True, maintainer=Dimitri Savineau , org.label-schema.name=CentOS Base Image, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, org.label-schema.build-date=20201204) 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[71738]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:11 smithi027 ceph-mon[67349]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.5 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.6 2024-09-17T09:14:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.2 2024-09-17T09:14:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.7 2024-09-17T09:14:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.3 2024-09-17T09:14:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.1 2024-09-17T09:14:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:11 smithi105 ceph-mon[70467]: osd.0 reported immediately failed by osd.4 2024-09-17T09:14:12.483 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:12 smithi027 podman[78119]: 2024-09-17 09:14:12.224699622 +0000 UTC m=+0.645689780 container cleanup 0249074d5c8d2c1c9bdd2e770b4bdd3ec71e27dde5eaf975d6d2a4859f665993 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0, RELEASE=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557) 2024-09-17T09:14:12.483 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:12 smithi027 bash[78119]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0 2024-09-17T09:14:12.483 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:12 smithi027 podman[78134]: 2024-09-17 09:14:12.434386522 +0000 UTC m=+0.717161256 container remove 0249074d5c8d2c1c9bdd2e770b4bdd3ec71e27dde5eaf975d6d2a4859f665993 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=-16.2.0, maintainer=Dimitri Savineau , GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS) 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[67349]: pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[67349]: osdmap e78: 8 total, 7 up, 8 in 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[67349]: osdmap e79: 8 total, 7 up, 8 in 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[71738]: pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[71738]: osdmap e78: 8 total, 7 up, 8 in 2024-09-17T09:14:13.155 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:12 smithi027 ceph-mon[71738]: osdmap e79: 8 total, 7 up, 8 in 2024-09-17T09:14:13.156 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:12 smithi027 podman[78236]: 2024-09-17 09:14:12.901969786 +0000 UTC m=+0.110828743 container create e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T09:14:13.156 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:12 smithi027 podman[78236]: 2024-09-17 09:14:12.807351097 +0000 UTC m=+0.016210033 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:13.156 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 podman[78236]: 2024-09-17 09:14:13.154741941 +0000 UTC m=+0.363600875 container init e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T09:14:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:12 smithi105 ceph-mon[70467]: pgmap v52: 161 pgs: 161 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:12 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:12 smithi105 ceph-mon[70467]: osdmap e78: 8 total, 7 up, 8 in 2024-09-17T09:14:13.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:12 smithi105 ceph-mon[70467]: osdmap e79: 8 total, 7 up, 8 in 2024-09-17T09:14:13.508 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 podman[78236]: 2024-09-17 09:14:13.158417357 +0000 UTC m=+0.367276273 container start e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:14:13.508 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 podman[78236]: 2024-09-17 09:14:13.255047668 +0000 UTC m=+0.463906600 container attach e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T09:14:13.508 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 podman[78236]: 2024-09-17 09:14:13.354802709 +0000 UTC m=+0.563661633 container died e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:14:14.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 podman[78259]: 2024-09-17 09:14:13.891664897 +0000 UTC m=+0.530680502 container remove e47faf77d3b7978701ee6f5a222b119a50b66efcea57670ae232893e672deea4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T09:14:14.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T09:14:14.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 systemd[1]: Stopped Ceph osd.0 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:14:14.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:13 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.0.service: Consumed 1.141s CPU time. 2024-09-17T09:14:14.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 systemd[1]: Starting Ceph osd.0 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:14:14.654 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.245612389 +0000 UTC m=+0.016500965 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:14.654 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.359730207 +0000 UTC m=+0.130618815 container create 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T09:14:14.654 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.58686397 +0000 UTC m=+0.357752550 container init 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b) 2024-09-17T09:14:14.654 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.590328666 +0000 UTC m=+0.361217233 container start 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T09:14:14.923 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.653503326 +0000 UTC m=+0.424391902 container attach 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T09:14:14.923 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:14:14.923 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:14:14.923 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:14:14.924 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:14:14.924 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:14:14.924 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:14:15.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-mon[67349]: pgmap v55: 161 pgs: 21 stale+active+clean, 140 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-mon[71738]: pgmap v55: 161 pgs: 21 stale+active+clean, 140 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:15.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:14:15.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:14:15.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate[78373]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 bash[78357]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T09:14:15.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:14 smithi027 podman[78357]: 2024-09-17 09:14:14.950074288 +0000 UTC m=+0.720962855 container died 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T09:14:15.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:14 smithi105 ceph-mon[70467]: pgmap v55: 161 pgs: 21 stale+active+clean, 140 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:15.891 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:15 smithi027 podman[78502]: 2024-09-17 09:14:15.549241427 +0000 UTC m=+0.588897384 container remove 58fb010320054f67e8d7dfa7b2783222ff75c15280daa48d07a07b63b110517f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-19e2a2b) 2024-09-17T09:14:15.891 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:15 smithi027 podman[78563]: 2024-09-17 09:14:15.660548283 +0000 UTC m=+0.016500164 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:15.891 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:15 smithi027 podman[78563]: 2024-09-17 09:14:15.783524006 +0000 UTC m=+0.139475878 container create 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T09:14:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-mon[67349]: pgmap v56: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:16.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-mon[71738]: pgmap v56: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:16.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:16.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:15 smithi027 podman[78563]: 2024-09-17 09:14:15.985798708 +0000 UTC m=+0.341750599 container init 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T09:14:16.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:15 smithi027 podman[78563]: 2024-09-17 09:14:15.989335355 +0000 UTC m=+0.345287229 container start 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T09:14:16.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-osd[78583]: -- 172.21.15.27:0/2036135593 <== mon.0 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56139be8f1e0 con 0x56139be2dc00 2024-09-17T09:14:16.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:16 smithi027 bash[78563]: 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d 2024-09-17T09:14:16.259 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:16 smithi027 systemd[1]: Started Ceph osd.0 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:14:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:16 smithi105 ceph-mon[70467]: pgmap v56: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:16.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:16 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:16.634 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:16 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:14:16.533+0000 7f8703c40640 -1 Falling back to public interface 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:17 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:17 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:17 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:17 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:17 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:18.428 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:18 smithi105 ceph-mon[70467]: pgmap v57: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:18.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:18 smithi027 ceph-mon[67349]: pgmap v57: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:18.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:18 smithi027 ceph-mon[71738]: pgmap v57: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 94 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:19.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:19 smithi027 ceph-mon[67349]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T09:14:19.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:19 smithi027 ceph-mon[71738]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T09:14:19.509 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:14:19.236+0000 7f8703c40640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T09:14:19.509 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:14:19.263+0000 7f8703c40640 -1 osd.0 77 log_to_monitors true 2024-09-17T09:14:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:19 smithi105 ceph-mon[70467]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T09:14:20.007 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:14:19 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:14:19.769+0000 7f86f79e2640 -1 osd.0 77 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:14:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:14:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: osdmap e80: 8 total, 7 up, 8 in 2024-09-17T09:14:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: pgmap v59: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: osdmap e80: 8 total, 7 up, 8 in 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: pgmap v59: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:20.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:20 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: osdmap e80: 8 total, 7 up, 8 in 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: from='osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917]' entity='osd.0' 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: pgmap v59: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:20 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:21.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917] boot 2024-09-17T09:14:21.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T09:14:21.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:21 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917] boot 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T09:14:21.934 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: osd.0 [v2:172.21.15.27:6802/436079917,v1:172.21.15.27:6803/436079917] boot 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T09:14:21.935 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:21 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:14:23.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:22 smithi105 ceph-mon[70467]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T09:14:23.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:22 smithi105 ceph-mon[70467]: pgmap v62: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:22 smithi027 ceph-mon[71738]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T09:14:23.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:22 smithi027 ceph-mon[71738]: pgmap v62: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:22 smithi027 ceph-mon[67349]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T09:14:23.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:22 smithi027 ceph-mon[67349]: pgmap v62: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:24.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:14:23 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:14:23.828+0000 7f5acd221640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: pgmap v63: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: pgmap v64: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[67349]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: pgmap v63: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: pgmap v64: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:24 smithi027 ceph-mon[71738]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: pgmap v63: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: pgmap v64: 161 pgs: 38 active+undersized, 20 active+undersized+degraded, 103 active+clean; 604 KiB data, 114 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:24 smithi105 ceph-mon[70467]: Upgrade: unsafe to stop osd(s) at this time (20 PGs are or would become offline) 2024-09-17T09:14:26.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:25 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded) 2024-09-17T09:14:26.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:25 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded) 2024-09-17T09:14:26.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:25 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 115/999 objects degraded (11.512%), 20 pgs degraded) 2024-09-17T09:14:27.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:26 smithi027 ceph-mon[67349]: pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:27.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:26 smithi027 ceph-mon[71738]: pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:26 smithi105 ceph-mon[70467]: pgmap v65: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:28 smithi027 ceph-mon[67349]: pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:29.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:28 smithi027 ceph-mon[71738]: pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:28 smithi105 ceph-mon[70467]: pgmap v66: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:31.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:30 smithi027 ceph-mon[67349]: pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:31.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:30 smithi027 ceph-mon[71738]: pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:30 smithi105 ceph-mon[70467]: pgmap v67: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:32 smithi027 ceph-mon[67349]: pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:33.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:32 smithi027 ceph-mon[71738]: pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:32 smithi105 ceph-mon[70467]: pgmap v68: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:34 smithi027 ceph-mon[67349]: pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:34 smithi027 ceph-mon[71738]: pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:34 smithi105 ceph-mon[70467]: pgmap v69: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:36 smithi027 ceph-mon[67349]: pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:36 smithi027 ceph-mon[71738]: pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:36 smithi105 ceph-mon[70467]: pgmap v70: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:39.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:38 smithi105 ceph-mon[70467]: pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:39.190 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:38 smithi027 ceph-mon[67349]: pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:39.190 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:38 smithi027 ceph-mon[71738]: pgmap v71: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:39.259 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:14:39.934 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 19s ago 25m 25.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (3m) 103s ago 25m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 103s ago 25m - - 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (5m) 103s ago 29m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (15m) 19s ago 31m 478M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (117s) 19s ago 31m 46.7M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (2m) 103s ago 29m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (95s) 19s ago 30m 35.8M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (4m) 19s ago 26m 16.3M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (4m) 103s ago 26m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (24s) 19s ago 29m 11.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (29m) 19s ago 29m 50.6M 4096M 16.2.0 24ecd6d5f14c 8d1a63b0894e 2024-09-17T09:14:39.935 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (28m) 19s ago 28m 46.4M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (28m) 19s ago 28m 51.2M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (28m) 103s ago 28m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (27m) 103s ago 27m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (27m) 103s ago 27m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (27m) 103s ago 27m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (4m) 103s ago 26m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (25m) 19s ago 25m 43.0M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:14:39.936 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (25m) 103s ago 25m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:14:40.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:39 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:39 smithi105 ceph-mon[70467]: Upgrade: osd.1 is safe to restart 2024-09-17T09:14:40.198 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.198 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.199 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[71738]: Upgrade: osd.1 is safe to restart 2024-09-17T09:14:40.199 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.199 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:14:40.199 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:39 smithi027 ceph-mon[67349]: Upgrade: osd.1 is safe to restart 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:40.342 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 7, 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 1 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 9, 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 6 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:14:40.343 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:14:40.697 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:14:40.697 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:14:40.697 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (1 remaining).", 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "1/8 daemons upgraded", 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:14:40.698 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:14:40.818 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[67349]: from='client.64234 172.21.15.27:0/3255506248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:40.819 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:40.820 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:40 smithi027 ceph-mon[71738]: from='client.64234 172.21.15.27:0/3255506248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='client.54276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='client.64222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: pgmap v72: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='client.54288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:41.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:40 smithi105 ceph-mon[70467]: from='client.64234 172.21.15.27:0/3255506248' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:41.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[67349]: Upgrade: Updating osd.1 2024-09-17T09:14:41.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[67349]: Deploying daemon osd.1 on smithi027 2024-09-17T09:14:41.775 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[67349]: from='client.54300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:41 smithi105 ceph-mon[70467]: Upgrade: Updating osd.1 2024-09-17T09:14:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:41 smithi105 ceph-mon[70467]: Deploying daemon osd.1 on smithi027 2024-09-17T09:14:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:41 smithi105 ceph-mon[70467]: from='client.54300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:42.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[71738]: Upgrade: Updating osd.1 2024-09-17T09:14:42.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[71738]: Deploying daemon osd.1 on smithi027 2024-09-17T09:14:42.051 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:41 smithi027 ceph-mon[71738]: from='client.54300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:14:42.990 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:42 smithi027 ceph-mon[67349]: pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:42.990 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:42 smithi027 ceph-mon[71738]: pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:43.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:42 smithi105 ceph-mon[70467]: pgmap v73: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:43.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:42 smithi027 systemd[1]: Stopping Ceph osd.1 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:14:43.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T09:14:43.132+0000 7f1824d41700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:14:43.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T09:14:43.133+0000 7f1824d41700 -1 osd.1 82 *** Got signal Terminated *** 2024-09-17T09:14:43.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1[31634]: debug 2024-09-17T09:14:43.133+0000 7f1824d41700 -1 osd.1 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:14:43.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 podman[83018]: 2024-09-17 09:14:43.164774265 +0000 UTC m=+0.134459540 container died 8d1a63b0894e2cab9f71fc6533b305e2fed5627e51cf735e3a475479901d5684 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, CEPH_POINT_RELEASE=-16.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Dimitri Savineau ) 2024-09-17T09:14:43.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 podman[83018]: 2024-09-17 09:14:43.533601547 +0000 UTC m=+0.503286813 container cleanup 8d1a63b0894e2cab9f71fc6533b305e2fed5627e51cf735e3a475479901d5684 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Dimitri Savineau , CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD) 2024-09-17T09:14:43.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 bash[83018]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1 2024-09-17T09:14:44.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:44.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:43 smithi105 ceph-mon[70467]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.253 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:43 smithi027 podman[83030]: 2024-09-17 09:14:43.793485207 +0000 UTC m=+0.624613625 container remove 8d1a63b0894e2cab9f71fc6533b305e2fed5627e51cf735e3a475479901d5684 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.1, ceph=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Base Image, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.build-date=20201204) 2024-09-17T09:14:44.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:44.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.254 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.255 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.256 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.257 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[67349]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 failed (root=default,host=smithi027) (connection refused reported by osd.2) 2024-09-17T09:14:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.3 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.2 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.0 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.4 2024-09-17T09:14:44.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.6 2024-09-17T09:14:44.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.5 2024-09-17T09:14:44.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:43 smithi027 ceph-mon[71738]: osd.1 reported immediately failed by osd.7 2024-09-17T09:14:44.508 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.252754837 +0000 UTC m=+0.100068637 container create 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T09:14:44.508 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.168934779 +0000 UTC m=+0.016248544 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:44.509 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.497703786 +0000 UTC m=+0.345017557 container init 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:14:44.509 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.501117863 +0000 UTC m=+0.348431619 container start 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2) 2024-09-17T09:14:44.776 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.597603864 +0000 UTC m=+0.444917632 container attach 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T09:14:44.776 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:44 smithi027 podman[83135]: 2024-09-17 09:14:44.702128185 +0000 UTC m=+0.549441948 container died 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:14:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:44 smithi105 ceph-mon[70467]: pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:44 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:44 smithi105 ceph-mon[70467]: osdmap e83: 8 total, 7 up, 8 in 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[71738]: pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[71738]: osdmap e83: 8 total, 7 up, 8 in 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[67349]: pgmap v74: 161 pgs: 161 active+clean; 604 KiB data, 119 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:14:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:44 smithi027 ceph-mon[67349]: osdmap e83: 8 total, 7 up, 8 in 2024-09-17T09:14:45.647 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 podman[83158]: 2024-09-17 09:14:45.368639471 +0000 UTC m=+0.660632965 container remove 2ad266d0f077c32d78b060d5e5b93952e656b50e3ca8bd896dc21b29e7009440 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True) 2024-09-17T09:14:45.648 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T09:14:45.648 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 systemd[1]: Stopped Ceph osd.1 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:14:45.648 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.1.service: Consumed 1.140s CPU time. 2024-09-17T09:14:45.648 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 systemd[1]: Starting Ceph osd.1 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:14:45.952 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:45 smithi027 ceph-mon[67349]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T09:14:45.952 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:45 smithi027 ceph-mon[71738]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T09:14:45.952 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 podman[83257]: 2024-09-17 09:14:45.728690391 +0000 UTC m=+0.016534825 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:45.953 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:45 smithi027 podman[83257]: 2024-09-17 09:14:45.828319248 +0000 UTC m=+0.116163682 container create 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T09:14:46.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 podman[83257]: 2024-09-17 09:14:46.072801426 +0000 UTC m=+0.360645854 container init 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T09:14:46.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 podman[83257]: 2024-09-17 09:14:46.076271548 +0000 UTC m=+0.364115982 container start 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T09:14:46.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 podman[83257]: 2024-09-17 09:14:46.15600718 +0000 UTC m=+0.443851608 container attach 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-17T09:14:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:45 smithi105 ceph-mon[70467]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:14:46.645 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate[83280]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 bash[83257]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T09:14:46.646 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:46 smithi027 podman[83257]: 2024-09-17 09:14:46.430035952 +0000 UTC m=+0.717880378 container died 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[67349]: pgmap v77: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 119/999 objects degraded (11.912%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[71738]: pgmap v77: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:14:47.002 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:46 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 119/999 objects degraded (11.912%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:47.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 podman[83406]: 2024-09-17 09:14:47.001441437 +0000 UTC m=+0.565082171 container remove 39ef88bbf9eefe05e4d3e487d74338df3a6fd6ffbcf0aed31244738dd515c722 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True) 2024-09-17T09:14:47.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 podman[83466]: 2024-09-17 09:14:47.113671205 +0000 UTC m=+0.016480036 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:14:47.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 podman[83466]: 2024-09-17 09:14:47.227568363 +0000 UTC m=+0.130377183 container create 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T09:14:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:46 smithi105 ceph-mon[70467]: pgmap v77: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:46 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:14:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:46 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 119/999 objects degraded (11.912%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 podman[83466]: 2024-09-17 09:14:47.44702043 +0000 UTC m=+0.349829260 container init 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:14:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 podman[83466]: 2024-09-17 09:14:47.45028869 +0000 UTC m=+0.353097518 container start 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b) 2024-09-17T09:14:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 ceph-osd[83486]: -- 172.21.15.27:0/4015764888 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564ecf8a91e0 con 0x564ed0628000 2024-09-17T09:14:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 bash[83466]: 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b 2024-09-17T09:14:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 systemd[1]: Started Ceph osd.1 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:14:48.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:14:47.988+0000 7fb0f97bb640 -1 Falling back to public interface 2024-09-17T09:14:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[67349]: pgmap v78: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:48 smithi027 ceph-mon[71738]: pgmap v78: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:48 smithi105 ceph-mon[70467]: pgmap v78: 161 pgs: 38 active+undersized, 7 peering, 18 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 119/999 objects degraded (11.912%) 2024-09-17T09:14:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:49 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:14:50.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:49 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:14:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:49 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:14:50.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-mon[67349]: pgmap v79: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:50.969 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-mon[67349]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T09:14:50.969 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-mon[71738]: pgmap v79: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:50.969 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-mon[71738]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T09:14:50.970 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:14:50.711+0000 7fb0f97bb640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T09:14:50.970 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:14:50.745+0000 7fb0f97bb640 -1 osd.1 82 log_to_monitors true 2024-09-17T09:14:50.970 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:14:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:14:50.836+0000 7fb0f1565640 -1 osd.1 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:14:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:50 smithi105 ceph-mon[70467]: pgmap v79: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 120 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:50 smithi105 ceph-mon[70467]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T09:14:52.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[71738]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:14:52.250 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[71738]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[71738]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[67349]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[67349]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[67349]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:52.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:51 smithi105 ceph-mon[70467]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:14:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:51 smithi105 ceph-mon[70467]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T09:14:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:51 smithi105 ceph-mon[70467]: from='osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:14:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:52.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: pgmap v81: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383] boot 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.178 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 130/999 objects degraded (13.013%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: pgmap v81: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383] boot 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.179 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:52 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 130/999 objects degraded (13.013%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: pgmap v81: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: osd.1 [v2:172.21.15.27:6810/4224221383,v1:172.21.15.27:6811/4224221383] boot 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:53.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:52 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 130/999 objects degraded (13.013%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:54.192 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:53 smithi027 ceph-mon[67349]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T09:14:54.192 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:53 smithi027 ceph-mon[71738]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T09:14:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:53 smithi105 ceph-mon[70467]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T09:14:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:54 smithi027 ceph-mon[67349]: pgmap v84: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:55.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:54 smithi027 ceph-mon[71738]: pgmap v84: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:54 smithi105 ceph-mon[70467]: pgmap v84: 161 pgs: 43 active+undersized, 20 active+undersized+degraded, 98 active+clean; 604 KiB data, 140 MiB used, 715 GiB / 715 GiB avail; 130/999 objects degraded (13.013%) 2024-09-17T09:14:56.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:14:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:14:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:14:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:14:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.267 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:14:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:14:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:14:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v85: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v86: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:14:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v87: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v88: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v89: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[67349]: pgmap v90: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v85: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v86: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:14:57.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v87: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v88: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v89: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:56 smithi027 ceph-mon[71738]: pgmap v90: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v85: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v86: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:14:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v87: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v88: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v89: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:56 smithi105 ceph-mon[70467]: pgmap v90: 161 pgs: 13 active+undersized, 7 active+undersized+degraded, 141 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 46/999 objects degraded (4.605%) 2024-09-17T09:14:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:57 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:58.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:57 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:57 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:14:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:58 smithi027 ceph-mon[67349]: pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-17T09:14:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:58 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded) 2024-09-17T09:14:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:58 smithi027 ceph-mon[71738]: pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-17T09:14:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:58 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded) 2024-09-17T09:14:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:58 smithi105 ceph-mon[70467]: pgmap v91: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 op/s 2024-09-17T09:14:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:58 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/999 objects degraded (4.605%), 7 pgs degraded) 2024-09-17T09:15:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:59 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:00.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:14:59 smithi027 ceph-mon[67349]: pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 5 op/s 2024-09-17T09:15:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:59 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:00.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:14:59 smithi027 ceph-mon[71738]: pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 5 op/s 2024-09-17T09:15:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:59 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:14:59 smithi105 ceph-mon[70467]: pgmap v92: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 5 op/s 2024-09-17T09:15:02.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:02 smithi027 ceph-mon[67349]: pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s 2024-09-17T09:15:02.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:02 smithi027 ceph-mon[71738]: pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s 2024-09-17T09:15:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:02 smithi105 ceph-mon[70467]: pgmap v93: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 4 op/s 2024-09-17T09:15:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:04 smithi027 ceph-mon[67349]: pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-17T09:15:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:04 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:04.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:04 smithi027 ceph-mon[71738]: pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-17T09:15:04.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:04 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:04 smithi105 ceph-mon[70467]: pgmap v94: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-09-17T09:15:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:04 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:06 smithi027 ceph-mon[67349]: pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-17T09:15:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:06 smithi027 ceph-mon[71738]: pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-17T09:15:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:06 smithi105 ceph-mon[70467]: pgmap v95: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 2 op/s 2024-09-17T09:15:08.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:08 smithi027 ceph-mon[67349]: pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T09:15:08.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:08 smithi027 ceph-mon[71738]: pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T09:15:08.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:08 smithi105 ceph-mon[70467]: pgmap v96: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-09-17T09:15:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:10 smithi027 ceph-mon[67349]: pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:10 smithi027 ceph-mon[71738]: pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:10 smithi105 ceph-mon[70467]: pgmap v97: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:11.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:15:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:12 smithi027 ceph-mon[67349]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:12.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:12 smithi027 ceph-mon[67349]: pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:12.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:12 smithi027 ceph-mon[71738]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:12.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:12 smithi027 ceph-mon[71738]: pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:12 smithi105 ceph-mon[70467]: from='client.64252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:12 smithi105 ceph-mon[70467]: pgmap v98: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (4m) 22s ago 26m 25.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (4m) 2m ago 26m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 2m ago 25m - - 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (5m) 2m ago 30m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (15m) 22s ago 31m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 22s ago 31m 48.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (3m) 2m ago 30m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (2m) 22s ago 30m 40.0M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (5m) 22s ago 27m 16.0M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:15:13.800 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (5m) 2m ago 26m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (58s) 22s ago 30m 49.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (26s) 22s ago 29m 11.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (29m) 22s ago 29m 47.9M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (29m) 22s ago 29m 51.5M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (28m) 2m ago 28m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (28m) 2m ago 28m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (28m) 2m ago 28m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (27m) 2m ago 27m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (4m) 2m ago 27m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (25m) 22s ago 25m 43.1M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:15:13.801 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (25m) 2m ago 25m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:15:14.590 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:14 smithi027 ceph-mon[71738]: pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:14.590 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:14 smithi027 ceph-mon[71738]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:14.590 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:14 smithi027 ceph-mon[67349]: pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:14.590 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:14 smithi027 ceph-mon[67349]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:14.616 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-17T09:15:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:14 smithi105 ceph-mon[70467]: pgmap v99: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:14 smithi105 ceph-mon[70467]: from='client.64258 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:16.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:16 smithi027 ceph-mon[67349]: pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:16.451 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:16 smithi027 ceph-mon[71738]: pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:16.761 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:15:16.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:16 smithi105 ceph-mon[70467]: pgmap v100: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:17.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-09-17T09:15:17.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:17 smithi027 ceph-mon[67349]: from='client.54324 172.21.15.27:0/3110775449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:17.664 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:17 smithi027 ceph-mon[71738]: from='client.54324 172.21.15.27:0/3110775449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:17 smithi105 ceph-mon[70467]: from='client.54324 172.21.15.27:0/3110775449' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:18.553 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:18 smithi027 ceph-mon[71738]: pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:18.554 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:18 smithi027 ceph-mon[67349]: pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:18 smithi105 ceph-mon[70467]: pgmap v101: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:20.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:20 smithi027 ceph-mon[67349]: from='client.54330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:20.506 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:20 smithi027 ceph-mon[67349]: pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:20.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:20 smithi027 ceph-mon[71738]: from='client.54330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:20.507 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:20 smithi027 ceph-mon[71738]: pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:20.544 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:15:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:20 smithi105 ceph-mon[70467]: from='client.54330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:20 smithi105 ceph-mon[70467]: pgmap v102: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:21.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:15:22.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:22 smithi027 ceph-mon[67349]: pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:22.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:22 smithi027 ceph-mon[71738]: pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:22.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:22 smithi105 ceph-mon[70467]: pgmap v103: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:23.229 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:15:23.229 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:15:23.229 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:15:23.230 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:15:23.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:15:24.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:24 smithi027 ceph-mon[67349]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:24.487 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:24 smithi027 ceph-mon[67349]: pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:24.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:24 smithi027 ceph-mon[71738]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:24.487 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:24 smithi027 ceph-mon[71738]: pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:24.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:24 smithi105 ceph-mon[70467]: from='client.54336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:24.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:24 smithi105 ceph-mon[70467]: pgmap v104: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:26.097 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:15:26.098 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:15:26.098 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:15:26.666 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:26 smithi027 ceph-mon[67349]: pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:26.666 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:26 smithi027 ceph-mon[67349]: from='client.54342 172.21.15.27:0/3461032685' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:15:26.666 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:26 smithi027 ceph-mon[71738]: pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:26.666 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:26 smithi027 ceph-mon[71738]: from='client.54342 172.21.15.27:0/3461032685' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:15:26.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-09-17T09:15:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:26 smithi105 ceph-mon[70467]: pgmap v105: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:26 smithi105 ceph-mon[70467]: from='client.54342 172.21.15.27:0/3461032685' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:15:28.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:28 smithi027 ceph-mon[67349]: pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:28.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:28 smithi027 ceph-mon[71738]: pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:28.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:28 smithi105 ceph-mon[70467]: pgmap v106: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:29.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:29 smithi027 ceph-mon[67349]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:29 smithi027 ceph-mon[71738]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:29.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:29 smithi105 ceph-mon[70467]: from='client.64288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:30.227 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[67349]: pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[71738]: pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:30.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:30.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:30 smithi105 ceph-mon[70467]: pgmap v107: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:30.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:30.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:30.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:31.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:31 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: pgmap v108: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: pgmap v109: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:31.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:31 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:15:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:15:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:15:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:33.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: Upgrade: osd.2 is safe to restart 2024-09-17T09:15:33.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[67349]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: Upgrade: osd.2 is safe to restart 2024-09-17T09:15:33.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:33 smithi027 ceph-mon[71738]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:15:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:15:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:15:33.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: Upgrade: osd.2 is safe to restart 2024-09-17T09:15:33.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:33 smithi105 ceph-mon[70467]: pgmap v110: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:34.840 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:15:34.917 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[71738]: Upgrade: Updating osd.2 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[71738]: Deploying daemon osd.2 on smithi027 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[67349]: Upgrade: Updating osd.2 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:34.918 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:34 smithi027 ceph-mon[67349]: Deploying daemon osd.2 on smithi027 2024-09-17T09:15:35.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:34 smithi105 ceph-mon[70467]: Upgrade: Updating osd.2 2024-09-17T09:15:35.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:35.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:15:35.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:35.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:34 smithi105 ceph-mon[70467]: Deploying daemon osd.2 on smithi027 2024-09-17T09:15:35.512 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:15:35.512 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 44s ago 26m 25.1M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (4m) 2m ago 26m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 2m ago 26m - - 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (6m) 2m ago 30m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (16m) 44s ago 32m 479M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (2m) 44s ago 32m 48.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (3m) 2m ago 30m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (2m) 44s ago 31m 40.0M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (5m) 44s ago 27m 16.0M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (5m) 2m ago 27m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (79s) 44s ago 30m 49.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (48s) 44s ago 30m 11.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (29m) 44s ago 29m 47.9M 4096M 16.2.0 24ecd6d5f14c fe3ccb01de18 2024-09-17T09:15:35.513 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (29m) 44s ago 29m 51.5M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (29m) 2m ago 29m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (28m) 2m ago 28m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (28m) 2m ago 28m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (28m) 2m ago 28m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (5m) 2m ago 27m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (26m) 44s ago 26m 43.1M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:15:35.514 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (26m) 2m ago 26m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:15:35.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:35 smithi027 ceph-mon[71738]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:35.932 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:35 smithi027 ceph-mon[71738]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:35.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:35 smithi027 ceph-mon[67349]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:35.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:35 smithi027 ceph-mon[67349]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:35.939 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 6, 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:15:35.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 8, 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 7 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:15:35.941 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:15:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:35 smithi105 ceph-mon[70467]: pgmap v111: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:35 smithi105 ceph-mon[70467]: from='client.64294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:36.280 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:15:36.280 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:15:36.280 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "2/8 daemons upgraded", 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:15:36.281 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:15:36.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[67349]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:36.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[67349]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:36.880 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[67349]: from='client.54372 172.21.15.27:0/2205913517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:36.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[71738]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:36.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[71738]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:36.881 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:36 smithi027 ceph-mon[71738]: from='client.54372 172.21.15.27:0/2205913517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:36 smithi105 ceph-mon[70467]: from='client.54360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:36 smithi105 ceph-mon[70467]: from='client.54366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:36 smithi105 ceph-mon[70467]: from='client.54372 172.21.15.27:0/2205913517' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:37.508 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 systemd[1]: Stopping Ceph osd.2 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:15:37.509 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T09:15:37.376+0000 7f40fa738700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:15:37.509 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T09:15:37.376+0000 7f40fa738700 -1 osd.2 87 *** Got signal Terminated *** 2024-09-17T09:15:37.509 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2[34040]: debug 2024-09-17T09:15:37.376+0000 7f40fa738700 -1 osd.2 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:15:37.509 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 podman[89369]: 2024-09-17 09:15:37.407694011 +0000 UTC m=+0.136402980 container died fe3ccb01de1868c6ed2762ad953aff7eff25b94da2e2df9542da4a51bfee6a10 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20201204) 2024-09-17T09:15:37.928 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 podman[89369]: 2024-09-17 09:15:37.701499678 +0000 UTC m=+0.430208641 container cleanup fe3ccb01de1868c6ed2762ad953aff7eff25b94da2e2df9542da4a51bfee6a10 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, maintainer=Dimitri Savineau , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.license=GPLv2, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20201204) 2024-09-17T09:15:37.928 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 bash[89369]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 failed (root=default,host=smithi027) (connection refused reported by osd.6) 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:37.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[67349]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 failed (root=default,host=smithi027) (connection refused reported by osd.6) 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:37.930 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:37.931 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:37 smithi027 ceph-mon[71738]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:38.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: from='client.54378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:15:38.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: pgmap v112: 161 pgs: 161 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:38.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 failed (root=default,host=smithi027) (connection refused reported by osd.6) 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.3 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.0 2024-09-17T09:15:38.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.1 2024-09-17T09:15:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.6 2024-09-17T09:15:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.7 2024-09-17T09:15:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.4 2024-09-17T09:15:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:37 smithi105 ceph-mon[70467]: osd.2 reported immediately failed by osd.5 2024-09-17T09:15:38.258 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:37 smithi027 podman[89380]: 2024-09-17 09:15:37.927774346 +0000 UTC m=+0.518238158 container remove fe3ccb01de1868c6ed2762ad953aff7eff25b94da2e2df9542da4a51bfee6a10 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.2, ceph=True, org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, RELEASE=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T09:15:38.687 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.276458888 +0000 UTC m=+0.016544824 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:15:38.688 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.378682693 +0000 UTC m=+0.118768627 container create e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T09:15:38.688 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.598249338 +0000 UTC m=+0.338335278 container init e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=reef-19e2a2b) 2024-09-17T09:15:38.688 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.601953046 +0000 UTC m=+0.342038986 container start e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:15:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:15:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[71738]: osdmap e88: 8 total, 7 up, 8 in 2024-09-17T09:15:39.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[71738]: osdmap e89: 8 total, 7 up, 8 in 2024-09-17T09:15:39.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:15:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[67349]: osdmap e88: 8 total, 7 up, 8 in 2024-09-17T09:15:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:38 smithi027 ceph-mon[67349]: osdmap e89: 8 total, 7 up, 8 in 2024-09-17T09:15:39.009 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.711632051 +0000 UTC m=+0.451718004 container attach e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T09:15:39.009 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:38 smithi027 podman[89481]: 2024-09-17 09:15:38.811020974 +0000 UTC m=+0.551106910 container died e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:15:39.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:38 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:15:39.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:38 smithi105 ceph-mon[70467]: osdmap e88: 8 total, 7 up, 8 in 2024-09-17T09:15:39.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:38 smithi105 ceph-mon[70467]: osdmap e89: 8 total, 7 up, 8 in 2024-09-17T09:15:39.689 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 podman[89503]: 2024-09-17 09:15:39.485631577 +0000 UTC m=+0.665253798 container remove e8cfc960f1c0f155e22fe11b521b80f0006b5646084add9812469f2bb7419673 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T09:15:39.689 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T09:15:39.689 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 systemd[1]: Stopped Ceph osd.2 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:15:39.689 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.2.service: Consumed 1.124s CPU time. 2024-09-17T09:15:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:39 smithi027 ceph-mon[67349]: pgmap v114: 161 pgs: 12 stale+active+clean, 149 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:39 smithi027 ceph-mon[71738]: pgmap v114: 161 pgs: 12 stale+active+clean, 149 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:40.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 systemd[1]: Starting Ceph osd.2 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:15:40.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 podman[89603]: 2024-09-17 09:15:39.82799148 +0000 UTC m=+0.016548658 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:15:40.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:39 smithi027 podman[89603]: 2024-09-17 09:15:39.936749714 +0000 UTC m=+0.125306908 container create 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b) 2024-09-17T09:15:40.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:39 smithi105 ceph-mon[70467]: pgmap v114: 161 pgs: 12 stale+active+clean, 149 active+clean; 604 KiB data, 141 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:40.484 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 podman[89603]: 2024-09-17 09:15:40.17287816 +0000 UTC m=+0.361435323 container init 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T09:15:40.485 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 podman[89603]: 2024-09-17 09:15:40.176240672 +0000 UTC m=+0.364797822 container start 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:15:40.485 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 podman[89603]: 2024-09-17 09:15:40.256327131 +0000 UTC m=+0.444884314 container attach 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:15:40.740 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate[89618]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 bash[89603]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T09:15:40.741 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:40 smithi027 podman[89603]: 2024-09-17 09:15:40.533598026 +0000 UTC m=+0.722155183 container died 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:15:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 61/999 objects degraded (6.106%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:41.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:40 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 61/999 objects degraded (6.106%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:41.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:40 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 61/999 objects degraded (6.106%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:41.421 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 podman[89747]: 2024-09-17 09:15:41.077073516 +0000 UTC m=+0.533509432 container remove 0db13c95b7394bafb369371924ebffd71763ce93903864c79bb642bf15e5ee37 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T09:15:41.422 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 podman[89806]: 2024-09-17 09:15:41.188840919 +0000 UTC m=+0.016251772 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:15:41.422 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 podman[89806]: 2024-09-17 09:15:41.311308561 +0000 UTC m=+0.138719459 container create 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T09:15:41.743 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 podman[89806]: 2024-09-17 09:15:41.538478869 +0000 UTC m=+0.365889722 container init 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:15:41.743 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 podman[89806]: 2024-09-17 09:15:41.542059789 +0000 UTC m=+0.369470641 container start 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:15:41.743 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-osd[89825]: -- 172.21.15.27:0/2417639578 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ecc873b1e0 con 0x55ecc94ba000 2024-09-17T09:15:41.744 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 bash[89806]: 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 2024-09-17T09:15:41.744 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:41 smithi027 systemd[1]: Started Ceph osd.2 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:15:41.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[67349]: pgmap v116: 161 pgs: 30 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 119 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 61/999 objects degraded (6.106%) 2024-09-17T09:15:41.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:41.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:41.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:41.745 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[71738]: pgmap v116: 161 pgs: 30 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 119 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 61/999 objects degraded (6.106%) 2024-09-17T09:15:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:41 smithi105 ceph-mon[70467]: pgmap v116: 161 pgs: 30 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 119 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 61/999 objects degraded (6.106%) 2024-09-17T09:15:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:42.376 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:15:42.079+0000 7f3bc9593640 -1 Falling back to public interface 2024-09-17T09:15:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:43 smithi027 ceph-mon[67349]: pgmap v117: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:43.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:43 smithi027 ceph-mon[71738]: pgmap v117: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:43 smithi105 ceph-mon[70467]: pgmap v117: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:45.053 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:15:44.784+0000 7f3bc9593640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T09:15:45.054 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:44 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:15:44.815+0000 7f3bc9593640 -1 osd.2 87 log_to_monitors true 2024-09-17T09:15:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[71738]: pgmap v118: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[71738]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[71738]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[67349]: pgmap v118: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:46.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[67349]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[67349]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.010 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:15:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:15:45.635+0000 7f3bc133d640 -1 osd.2 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:15:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:45 smithi105 ceph-mon[70467]: pgmap v118: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 146 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:45 smithi105 ceph-mon[70467]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:45 smithi105 ceph-mon[70467]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:15:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:45 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:45 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.995 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: osdmap e90: 8 total, 7 up, 8 in 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.996 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:15:46.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: osdmap e90: 8 total, 7 up, 8 in 2024-09-17T09:15:46.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:46.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:46.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:46.997 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: osdmap e90: 8 total, 7 up, 8 in 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: from='osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:48.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: pgmap v120: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119] boot 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 91/999 objects degraded (9.109%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: pgmap v120: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119] boot 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T09:15:48.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:15:48.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:47 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 91/999 objects degraded (9.109%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: pgmap v120: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: osd.2 [v2:172.21.15.27:6818/2189012119,v1:172.21.15.27:6819/2189012119] boot 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:15:48.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:47 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 91/999 objects degraded (9.109%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:15:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:48 smithi027 ceph-mon[67349]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T09:15:49.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:48 smithi027 ceph-mon[71738]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T09:15:49.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:48 smithi105 ceph-mon[70467]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T09:15:50.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: pgmap v123: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:15:50.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: pgmap v123: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:15:50.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:15:50.014 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.017 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.018 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:49 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: pgmap v123: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:50.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:15:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:15:50.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:15:50.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:49 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v124: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v125: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v126: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v127: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v128: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[71738]: pgmap v129: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:51.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v124: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v125: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v126: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v127: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v128: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:51.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:50 smithi027 ceph-mon[67349]: pgmap v129: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v124: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v125: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:15:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v126: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v127: 161 pgs: 34 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 91/999 objects degraded (9.109%) 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v128: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:50 smithi105 ceph-mon[70467]: pgmap v129: 161 pgs: 30 active+undersized, 12 active+undersized+degraded, 119 active+clean; 604 KiB data, 166 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:15:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:51 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 12 pgs degraded) 2024-09-17T09:15:52.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:51 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 12 pgs degraded) 2024-09-17T09:15:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:51 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 12 pgs degraded) 2024-09-17T09:15:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:52 smithi027 ceph-mon[67349]: pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:53.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:52 smithi027 ceph-mon[71738]: pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:52 smithi105 ceph-mon[70467]: pgmap v130: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:54 smithi027 ceph-mon[67349]: pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:54 smithi027 ceph-mon[71738]: pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:54 smithi105 ceph-mon[70467]: pgmap v131: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:15:57.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:56 smithi027 ceph-mon[71738]: pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:57.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:56 smithi027 ceph-mon[67349]: pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:57.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:56 smithi105 ceph-mon[70467]: pgmap v132: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:59.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:15:58 smithi027 ceph-mon[71738]: pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:59.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:15:58 smithi027 ceph-mon[67349]: pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:15:59.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:15:58 smithi105 ceph-mon[70467]: pgmap v133: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:00 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:00.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:00 smithi105 ceph-mon[70467]: pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:00 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:00 smithi027 ceph-mon[71738]: pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:00 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:00 smithi027 ceph-mon[67349]: pgmap v134: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:02 smithi105 ceph-mon[70467]: pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:03.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:02 smithi027 ceph-mon[71738]: pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:03.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:02 smithi027 ceph-mon[67349]: pgmap v135: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:04 smithi105 ceph-mon[70467]: pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:04.865 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:04 smithi027 ceph-mon[67349]: pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:04.866 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:04 smithi027 ceph-mon[71738]: pgmap v136: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:06 smithi105 ceph-mon[70467]: pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:06.933 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:06 smithi027 ceph-mon[67349]: pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:06.933 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:06 smithi027 ceph-mon[71738]: pgmap v137: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:07.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:16:07.773 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:07 smithi027 ceph-mon[67349]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:07.773 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:07 smithi027 ceph-mon[71738]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:07.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:07 smithi105 ceph-mon[70467]: from='client.54390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:08.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:08 smithi027 ceph-mon[67349]: pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:08.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:08 smithi027 ceph-mon[71738]: pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:08.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:08 smithi105 ceph-mon[70467]: pgmap v138: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 23s ago 27m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (5m) 3m ago 27m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 3m ago 26m - - 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (6m) 3m ago 31m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (16m) 23s ago 32m 481M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:16:09.197 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (3m) 23s ago 32m 51.6M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (4m) 3m ago 31m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (3m) 23s ago 31m 42.6M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 23s ago 27m 15.9M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (6m) 3m ago 27m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (113s) 23s ago 30m 51.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (82s) 23s ago 30m 43.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (28s) 23s ago 30m 11.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (29m) 23s ago 29m 51.7M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (29m) 3m ago 29m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (29m) 3m ago 29m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (29m) 3m ago 29m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (28m) 3m ago 28m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (5m) 3m ago 28m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:16:09.198 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (26m) 23s ago 26m 43.3M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:16:09.199 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (26m) 3m ago 26m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:16:09.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-09-17T09:16:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:10 smithi027 ceph-mon[67349]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:10 smithi027 ceph-mon[67349]: pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:10 smithi027 ceph-mon[71738]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:10.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:10 smithi027 ceph-mon[71738]: pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:10 smithi105 ceph-mon[70467]: from='client.64336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:10 smithi105 ceph-mon[70467]: pgmap v139: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:11.877 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:16:12.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:12 smithi027 ceph-mon[67349]: pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:12.640 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:12 smithi027 ceph-mon[67349]: from='client.54402 172.21.15.27:0/367994072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:12.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:12 smithi027 ceph-mon[71738]: pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:12.641 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:12 smithi027 ceph-mon[71738]: from='client.54402 172.21.15.27:0/367994072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:12.664 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-09-17T09:16:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:12 smithi105 ceph-mon[70467]: pgmap v140: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:12 smithi105 ceph-mon[70467]: from='client.54402 172.21.15.27:0/367994072' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:14.994 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:14 smithi027 ceph-mon[67349]: pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:14.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:14 smithi027 ceph-mon[71738]: pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:15.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:14 smithi105 ceph-mon[70467]: pgmap v141: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:15.724 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:15 smithi027 ceph-mon[67349]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:15.724 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:15 smithi027 ceph-mon[71738]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:16.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:15 smithi105 ceph-mon[70467]: from='client.64348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:16.074 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:16:16.654 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:16 smithi027 ceph-mon[67349]: pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:16.654 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:16 smithi027 ceph-mon[71738]: pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:16.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:16:17.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:16 smithi105 ceph-mon[70467]: pgmap v142: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:16:18.716 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:16:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:18 smithi027 ceph-mon[67349]: pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:18 smithi027 ceph-mon[71738]: pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:18 smithi105 ceph-mon[70467]: pgmap v143: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:19.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:16:19.607 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:19 smithi027 ceph-mon[67349]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:19.607 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:19 smithi027 ceph-mon[71738]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:20.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:19 smithi105 ceph-mon[70467]: from='client.64354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:20.593 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:20 smithi027 ceph-mon[67349]: pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:20.593 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:20 smithi027 ceph-mon[71738]: pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:21.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:20 smithi105 ceph-mon[70467]: pgmap v144: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:21.353 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:16:21.354 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:16:21.354 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in unknown state 2024-09-17T09:16:21.667 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:21 smithi027 ceph-mon[67349]: from='client.64360 172.21.15.27:0/2629368364' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:16:21.667 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:21 smithi027 ceph-mon[71738]: from='client.64360 172.21.15.27:0/2629368364' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:16:22.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:21 smithi105 ceph-mon[70467]: from='client.64360 172.21.15.27:0/2629368364' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:16:22.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-09-17T09:16:22.705 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:22 smithi027 ceph-mon[67349]: pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:22.705 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:22 smithi027 ceph-mon[71738]: pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:23.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:22 smithi105 ceph-mon[70467]: pgmap v145: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:24.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:24 smithi027 ceph-mon[67349]: pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:24.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:24 smithi027 ceph-mon[71738]: pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:24 smithi105 ceph-mon[70467]: pgmap v146: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:25.549 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[67349]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[71738]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:25.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:25 smithi105 ceph-mon[70467]: from='client.64366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:26.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:26.221 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:16:27.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:27.225 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:27.227 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.228 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:26 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: pgmap v147: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: pgmap v148: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: pgmap v149: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:26 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.207 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.208 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.209 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.209 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.209 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:28 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: pgmap v150: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:29.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:28 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:29.555 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:16:30.170 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[67349]: Upgrade: osd.3 is safe to restart 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:30.171 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:29 smithi027 ceph-mon[71738]: Upgrade: osd.3 is safe to restart 2024-09-17T09:16:30.235 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:16:30.235 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (5m) 44s ago 27m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (5m) 3m ago 27m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 3m ago 27m - - 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (6m) 3m ago 31m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (17m) 44s ago 32m 481M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (3m) 44s ago 33m 51.6M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (4m) 3m ago 31m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (3m) 44s ago 31m 42.6M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (6m) 44s ago 28m 15.9M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (6m) 3m ago 28m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (2m) 44s ago 31m 51.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (103s) 44s ago 30m 43.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:16:30.236 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (49s) 44s ago 30m 11.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (30m) 44s ago 30m 51.7M 4096M 16.2.0 24ecd6d5f14c 355c78923aba 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (30m) 3m ago 30m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (29m) 3m ago 29m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (29m) 3m ago 29m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (29m) 3m ago 29m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (6m) 3m ago 28m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (27m) 44s ago 27m 43.3M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:16:30.237 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (27m) 3m ago 27m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:16:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:16:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:16:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:16:30.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:16:30.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:16:30.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:16:30.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:29 smithi105 ceph-mon[70467]: Upgrade: osd.3 is safe to restart 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 5, 2024-09-17T09:16:30.658 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 7, 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 8 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:16:30.659 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "3/8 daemons upgraded", 2024-09-17T09:16:31.003 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:16:31.004 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:16:31.004 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:31.228 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[71738]: from='client.64390 172.21.15.27:0/1214600145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:31.229 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:30 smithi027 ceph-mon[67349]: from='client.64390 172.21.15.27:0/1214600145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='client.54432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: pgmap v151: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='client.54438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:31.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:16:31.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:31.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:30 smithi105 ceph-mon[70467]: from='client.64390 172.21.15.27:0/1214600145' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[71738]: Upgrade: Updating osd.3 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[71738]: Deploying daemon osd.3 on smithi027 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[71738]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[67349]: Upgrade: Updating osd.3 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[67349]: Deploying daemon osd.3 on smithi027 2024-09-17T09:16:32.185 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:31 smithi027 ceph-mon[67349]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:31 smithi105 ceph-mon[70467]: Upgrade: Updating osd.3 2024-09-17T09:16:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:31 smithi105 ceph-mon[70467]: Deploying daemon osd.3 on smithi027 2024-09-17T09:16:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:31 smithi105 ceph-mon[70467]: from='client.54444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:33.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:32 smithi027 ceph-mon[67349]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:33.216 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:32 smithi027 ceph-mon[67349]: pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:33.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:32 smithi027 ceph-mon[71738]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:33.216 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:32 smithi027 ceph-mon[71738]: pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:32 smithi105 ceph-mon[70467]: from='client.64396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:16:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:32 smithi105 ceph-mon[70467]: pgmap v152: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:33.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 systemd[1]: Stopping Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:16:33.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T09:16:33.380+0000 7f54854f3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:16:33.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T09:16:33.380+0000 7f54854f3700 -1 osd.3 92 *** Got signal Terminated *** 2024-09-17T09:16:33.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3[36453]: debug 2024-09-17T09:16:33.380+0000 7f54854f3700 -1 osd.3 92 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:16:33.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 podman[95830]: 2024-09-17 09:16:33.412617805 +0000 UTC m=+0.157246517 container died 355c78923abaabad00e1b7017379209068797231ceff0cb63ce76401bbd4b98c (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3, RELEASE=HEAD, maintainer=Dimitri Savineau , GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0) 2024-09-17T09:16:34.008 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 podman[95830]: 2024-09-17 09:16:33.865130611 +0000 UTC m=+0.609759315 container cleanup 355c78923abaabad00e1b7017379209068797231ceff0cb63ce76401bbd4b98c (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T09:16:34.008 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:33 smithi027 bash[95830]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3 2024-09-17T09:16:34.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 failed (root=default,host=smithi027) (connection refused reported by osd.1) 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[67349]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 failed (root=default,host=smithi027) (connection refused reported by osd.1) 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:33 smithi027 ceph-mon[71738]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 failed (root=default,host=smithi027) (connection refused reported by osd.1) 2024-09-17T09:16:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.1 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.0 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.2 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.4 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.5 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.7 2024-09-17T09:16:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:33 smithi105 ceph-mon[70467]: osd.3 reported immediately failed by osd.6 2024-09-17T09:16:34.508 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95842]: 2024-09-17 09:16:34.20054126 +0000 UTC m=+0.782588052 container remove 355c78923abaabad00e1b7017379209068797231ceff0cb63ce76401bbd4b98c (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.3, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, maintainer=Dimitri Savineau , org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=HEAD, ceph=True) 2024-09-17T09:16:34.921 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95944]: 2024-09-17 09:16:34.651612303 +0000 UTC m=+0.108989198 container create ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T09:16:34.922 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95944]: 2024-09-17 09:16:34.559288712 +0000 UTC m=+0.016665616 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:16:34.922 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95944]: 2024-09-17 09:16:34.870985468 +0000 UTC m=+0.328362372 container init ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T09:16:34.922 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95944]: 2024-09-17 09:16:34.874368343 +0000 UTC m=+0.331745244 container start ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T09:16:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[71738]: pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:16:35.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[71738]: osdmap e93: 8 total, 7 up, 8 in 2024-09-17T09:16:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[67349]: pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:35.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:16:35.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:34 smithi027 ceph-mon[67349]: osdmap e93: 8 total, 7 up, 8 in 2024-09-17T09:16:35.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:34 smithi027 podman[95944]: 2024-09-17 09:16:34.96480063 +0000 UTC m=+0.422177541 container attach ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True) 2024-09-17T09:16:35.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:35 smithi027 podman[95944]: 2024-09-17 09:16:35.074049702 +0000 UTC m=+0.531426604 container died ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T09:16:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:34 smithi105 ceph-mon[70467]: pgmap v153: 161 pgs: 161 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:34 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:16:35.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:34 smithi105 ceph-mon[70467]: osdmap e93: 8 total, 7 up, 8 in 2024-09-17T09:16:36.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:35 smithi027 ceph-mon[67349]: osdmap e94: 8 total, 7 up, 8 in 2024-09-17T09:16:36.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:35 smithi027 ceph-mon[71738]: osdmap e94: 8 total, 7 up, 8 in 2024-09-17T09:16:36.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:35 smithi027 podman[95967]: 2024-09-17 09:16:35.850785941 +0000 UTC m=+0.770843639 container remove ff48cb079f8a421b5b6243750461afe1f88227e8a4b7af28c848c25c1f50261b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T09:16:36.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:35 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T09:16:36.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:35 smithi027 systemd[1]: Stopped Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:16:36.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:35 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service: Consumed 1.143s CPU time. 2024-09-17T09:16:36.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 systemd[1]: Starting Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:16:36.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:35 smithi105 ceph-mon[70467]: osdmap e94: 8 total, 7 up, 8 in 2024-09-17T09:16:36.597 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.209676895 +0000 UTC m=+0.016653259 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:16:36.597 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.310708028 +0000 UTC m=+0.117684438 container create 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:16:36.597 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.513210975 +0000 UTC m=+0.320187342 container init 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T09:16:36.597 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.517291876 +0000 UTC m=+0.324268244 container start 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T09:16:36.597 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.596367631 +0000 UTC m=+0.403344020 container attach 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:16:36.848 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:16:36.849 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:16:36.849 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:16:36.849 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:16:36.850 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:16:36.850 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:16:36.850 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:16:36.850 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:16:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[67349]: pgmap v156: 161 pgs: 27 active+undersized, 14 peering, 5 stale+active+clean, 12 active+undersized+degraded, 103 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 60/999 objects degraded (6.006%) 2024-09-17T09:16:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T09:16:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 60/999 objects degraded (6.006%), 12 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[71738]: pgmap v156: 161 pgs: 27 active+undersized, 14 peering, 5 stale+active+clean, 12 active+undersized+degraded, 103 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 60/999 objects degraded (6.006%) 2024-09-17T09:16:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T09:16:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 60/999 objects degraded (6.006%), 12 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate[96083]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 bash[96068]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T09:16:37.259 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:36 smithi027 podman[96068]: 2024-09-17 09:16:36.874623925 +0000 UTC m=+0.681600298 container died 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T09:16:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:36 smithi105 ceph-mon[70467]: pgmap v156: 161 pgs: 27 active+undersized, 14 peering, 5 stale+active+clean, 12 active+undersized+degraded, 103 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 60/999 objects degraded (6.006%) 2024-09-17T09:16:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:36 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-17T09:16:37.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:36 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 60/999 objects degraded (6.006%), 12 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:37.953 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:37 smithi027 podman[96196]: 2024-09-17 09:16:37.609611111 +0000 UTC m=+0.725292695 container remove 36acd2db59a6c221b7f84ac579ee848d7e48eab74a86c14a106162706effffe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:16:37.953 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:37 smithi027 podman[96256]: 2024-09-17 09:16:37.827146654 +0000 UTC m=+0.110892161 container create 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:16:37.953 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:37 smithi027 podman[96256]: 2024-09-17 09:16:37.7338736 +0000 UTC m=+0.017619116 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:16:38.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 podman[96256]: 2024-09-17 09:16:38.054695869 +0000 UTC m=+0.338441380 container init 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T09:16:38.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 podman[96256]: 2024-09-17 09:16:38.057976809 +0000 UTC m=+0.341722309 container start 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-19e2a2b, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-17T09:16:38.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-osd[96275]: -- 172.21.15.27:0/1315623764 <== mon.1 v2:172.21.15.27:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bf86ef11e0 con 0x55bf87c70000 2024-09-17T09:16:38.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 bash[96256]: 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 2024-09-17T09:16:38.258 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 systemd[1]: Started Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:16:38.931 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:16:38.597+0000 7ff8f1d23640 -1 Falling back to public interface 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[67349]: pgmap v157: 161 pgs: 35 active+undersized, 14 peering, 17 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 89/999 objects degraded (8.909%) 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[71738]: pgmap v157: 161 pgs: 35 active+undersized, 14 peering, 17 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 89/999 objects degraded (8.909%) 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:38 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:38 smithi105 ceph-mon[70467]: pgmap v157: 161 pgs: 35 active+undersized, 14 peering, 17 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 89/999 objects degraded (8.909%) 2024-09-17T09:16:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:38 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:38 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:39.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:38 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:39 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:16:40.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:39 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:16:40.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:39 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:16:41.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:40 smithi027 ceph-mon[67349]: pgmap v158: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:41.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:40 smithi027 ceph-mon[71738]: pgmap v158: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:41.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:40 smithi105 ceph-mon[70467]: pgmap v158: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:41.649 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:16:41.328+0000 7ff8f1d23640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T09:16:41.649 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:16:41.370+0000 7ff8f1d23640 -1 osd.3 92 log_to_monitors true 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[67349]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[71738]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:41 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:41 smithi105 ceph-mon[70467]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T09:16:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:41 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:42.756 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:16:42.476+0000 7ff8e5ac5640 -1 osd.3 92 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: pgmap v159: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 136/999 objects degraded (13.614%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: pgmap v159: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' 2024-09-17T09:16:43.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:42 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 136/999 objects degraded (13.614%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: pgmap v159: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 167 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:16:43.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T09:16:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2024-09-17T09:16:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: from='osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837]' entity='osd.3' 2024-09-17T09:16:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:42 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 136/999 objects degraded (13.614%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:16:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[67349]: osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837] boot 2024-09-17T09:16:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[67349]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T09:16:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:16:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:16:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[71738]: osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837] boot 2024-09-17T09:16:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[71738]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T09:16:44.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:16:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:43 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:16:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:43 smithi105 ceph-mon[70467]: osd.3 [v2:172.21.15.27:6826/1183328837,v1:172.21.15.27:6827/1183328837] boot 2024-09-17T09:16:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:43 smithi105 ceph-mon[70467]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T09:16:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:16:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:44 smithi027 ceph-mon[67349]: pgmap v162: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:45.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:44 smithi027 ceph-mon[67349]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T09:16:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:44 smithi027 ceph-mon[71738]: pgmap v162: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:45.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:44 smithi027 ceph-mon[71738]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T09:16:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:44 smithi105 ceph-mon[70467]: pgmap v162: 161 pgs: 43 active+undersized, 23 active+undersized+degraded, 95 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 136/999 objects degraded (13.614%) 2024-09-17T09:16:45.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:44 smithi105 ceph-mon[70467]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T09:16:46.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:16:45 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:16:45.914+0000 7f5acd221640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-17T09:16:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: pgmap v164: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: pgmap v165: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[67349]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: pgmap v164: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: pgmap v165: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:46 smithi027 ceph-mon[71738]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-17T09:16:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: pgmap v164: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: pgmap v165: 161 pgs: 8 peering, 22 active+undersized, 13 active+undersized+degraded, 118 active+clean; 604 KiB data, 187 MiB used, 715 GiB / 715 GiB avail; 80/999 objects degraded (8.008%) 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:16:47.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:46 smithi105 ceph-mon[70467]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-17T09:16:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:47 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:48.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:47 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:48.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:47 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T09:16:49.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:48 smithi027 ceph-mon[67349]: pgmap v166: 161 pgs: 8 peering, 153 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:49.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:48 smithi027 ceph-mon[71738]: pgmap v166: 161 pgs: 8 peering, 153 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:49.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:48 smithi105 ceph-mon[70467]: pgmap v166: 161 pgs: 8 peering, 153 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:49 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded) 2024-09-17T09:16:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:49 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded) 2024-09-17T09:16:50.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:49 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 80/999 objects degraded (8.008%), 13 pgs degraded) 2024-09-17T09:16:51.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:50 smithi027 ceph-mon[67349]: pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:51.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:50 smithi027 ceph-mon[71738]: pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:51.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:50 smithi105 ceph-mon[70467]: pgmap v167: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:52 smithi027 ceph-mon[71738]: pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:53.396 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:52 smithi027 ceph-mon[67349]: pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:53.397 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:52 smithi105 ceph-mon[70467]: pgmap v168: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:55.060 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:54 smithi027 ceph-mon[67349]: pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:55.061 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:54 smithi027 ceph-mon[71738]: pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:55.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:54 smithi105 ceph-mon[70467]: pgmap v169: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:57.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:56 smithi027 ceph-mon[71738]: pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:56 smithi027 ceph-mon[67349]: pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:56 smithi105 ceph-mon[70467]: pgmap v170: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:16:58 smithi027 ceph-mon[71738]: pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:16:58 smithi027 ceph-mon[67349]: pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:16:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:16:58 smithi105 ceph-mon[70467]: pgmap v171: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:00 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:00 smithi027 ceph-mon[67349]: pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:00 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:00 smithi027 ceph-mon[71738]: pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:00 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:00 smithi105 ceph-mon[70467]: pgmap v172: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:01.358 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:01 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.358 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:01 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.358 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:01 smithi105 ceph-mon[70467]: Upgrade: osd.4 is safe to restart 2024-09-17T09:17:01.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[71738]: Upgrade: osd.4 is safe to restart 2024-09-17T09:17:01.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.369 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:17:01.370 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:01 smithi027 ceph-mon[67349]: Upgrade: osd.4 is safe to restart 2024-09-17T09:17:01.370 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:17:02.029 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:17:02.029 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (6m) 20s ago 28m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:17:02.029 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (6m) 4m ago 27m 84.7M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 unknown 4m ago 27m - - 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (7m) 4m ago 32m 434M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (17m) 20s ago 33m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (4m) 20s ago 33m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (5m) 4m ago 32m 42.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:17:02.030 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (3m) 20s ago 32m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (7m) 20s ago 28m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (7m) 4m ago 28m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (2m) 20s ago 31m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (2m) 20s ago 31m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (80s) 20s ago 31m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (24s) 20s ago 30m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:17:02.031 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (30m) 4m ago 30m 54.0M 4096M 16.2.0 24ecd6d5f14c e92ae516561a 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (30m) 4m ago 30m 47.5M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (29m) 4m ago 29m 48.8M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (29m) 4m ago 29m 53.0M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (6m) 4m ago 29m 51.4M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (27m) 20s ago 27m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:17:02.032 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (27m) 4m ago 27m 44.0M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:17:02.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 4, 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 4 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:02.432 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:17:02.433 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 6, 2024-09-17T09:17:02.433 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 9 2024-09-17T09:17:02.433 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:17:02.433 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:17:02.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[67349]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[67349]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.433 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[67349]: pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:02.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[71738]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[71738]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:02 smithi027 ceph-mon[71738]: pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:02.507 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:02 smithi105 ceph-mon[70467]: from='client.64408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.507 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:02 smithi105 ceph-mon[70467]: from='client.64414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:02.507 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:02 smithi105 ceph-mon[70467]: pgmap v173: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:02.769 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:17:02.769 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:17:02.769 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:17:02.769 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T09:17:02.770 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:17:02.770 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "4/8 daemons upgraded", 2024-09-17T09:17:02.770 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:17:02.770 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:17:02.770 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:17:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: Upgrade: Updating osd.4 2024-09-17T09:17:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:17:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: Deploying daemon osd.4 on smithi105 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='client.64426 172.21.15.27:0/2543708389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[67349]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: Upgrade: Updating osd.4 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:03.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: Deploying daemon osd.4 on smithi105 2024-09-17T09:17:03.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='client.64426 172.21.15.27:0/2543708389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:03.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:03 smithi027 ceph-mon[71738]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:03.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='client.64420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:03.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: Upgrade: Updating osd.4 2024-09-17T09:17:03.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:17:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: Deploying daemon osd.4 on smithi105 2024-09-17T09:17:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='client.64426 172.21.15.27:0/2543708389' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:03 smithi105 ceph-mon[70467]: from='client.64432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:04.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:04 smithi027 ceph-mon[67349]: pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:04.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:04 smithi027 ceph-mon[71738]: pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:04.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:04 smithi105 ceph-mon[70467]: pgmap v174: 161 pgs: 161 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:05.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 systemd[1]: Stopping Ceph osd.4 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:17:05.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T09:17:05.279+0000 7fa331274700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:17:05.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T09:17:05.279+0000 7fa331274700 -1 osd.4 97 *** Got signal Terminated *** 2024-09-17T09:17:05.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4[26497]: debug 2024-09-17T09:17:05.279+0000 7fa331274700 -1 osd.4 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:17:05.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 podman[81360]: 2024-09-17 09:17:05.321615688 +0000 UTC m=+0.131132811 container died e92ae516561abe009c214761960eaf1c7a56b299f4414bf87c6204ba6c11f8ab (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Base Image, ceph=True, CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20201204, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Dimitri Savineau ) 2024-09-17T09:17:05.982 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 podman[81360]: 2024-09-17 09:17:05.672029342 +0000 UTC m=+0.481546454 container cleanup e92ae516561abe009c214761960eaf1c7a56b299f4414bf87c6204ba6c11f8ab (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4, org.label-schema.vendor=CentOS, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-16.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image) 2024-09-17T09:17:05.982 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 bash[81360]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 failed (root=default,host=smithi105) (connection refused reported by osd.7) 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:05.983 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:05.984 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:05.984 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:05.984 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:05.984 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:05.984 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:05 smithi105 ceph-mon[70467]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 failed (root=default,host=smithi105) (connection refused reported by osd.7) 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:06.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[67349]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 failed (root=default,host=smithi105) (connection refused reported by osd.7) 2024-09-17T09:17:06.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.7 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.0 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.1 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.3 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.2 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.6 2024-09-17T09:17:06.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:05 smithi027 ceph-mon[71738]: osd.4 reported immediately failed by osd.5 2024-09-17T09:17:06.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:05 smithi105 podman[81372]: 2024-09-17 09:17:05.981974735 +0000 UTC m=+0.661926832 container remove e92ae516561abe009c214761960eaf1c7a56b299f4414bf87c6204ba6c11f8ab (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.4, ceph=True, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, RELEASE=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_BRANCH=HEAD) 2024-09-17T09:17:06.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:06 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:06.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:06 smithi105 ceph-mon[70467]: osdmap e98: 8 total, 7 up, 8 in 2024-09-17T09:17:06.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:06 smithi105 ceph-mon[70467]: pgmap v176: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:06.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:06 smithi105 ceph-mon[70467]: osdmap e99: 8 total, 7 up, 8 in 2024-09-17T09:17:06.714 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:06 smithi105 podman[81478]: 2024-09-17 09:17:06.432957021 +0000 UTC m=+0.102717696 container create d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True) 2024-09-17T09:17:06.714 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:06 smithi105 podman[81478]: 2024-09-17 09:17:06.346336369 +0000 UTC m=+0.016097054 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[67349]: osdmap e98: 8 total, 7 up, 8 in 2024-09-17T09:17:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[67349]: pgmap v176: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[67349]: osdmap e99: 8 total, 7 up, 8 in 2024-09-17T09:17:06.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:06.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[71738]: osdmap e98: 8 total, 7 up, 8 in 2024-09-17T09:17:06.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[71738]: pgmap v176: 161 pgs: 6 peering, 20 stale+active+clean, 135 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:06.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:06 smithi027 ceph-mon[71738]: osdmap e99: 8 total, 7 up, 8 in 2024-09-17T09:17:07.010 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:06 smithi105 podman[81478]: 2024-09-17 09:17:06.8189166 +0000 UTC m=+0.488677284 container init d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:17:07.011 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:06 smithi105 podman[81478]: 2024-09-17 09:17:06.822233562 +0000 UTC m=+0.491994238 container start d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T09:17:07.011 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:06 smithi105 podman[81478]: 2024-09-17 09:17:06.893830044 +0000 UTC m=+0.563590733 container attach d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T09:17:07.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 podman[81478]: 2024-09-17 09:17:07.010336795 +0000 UTC m=+0.680097479 container died d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T09:17:07.556 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 podman[81512]: 2024-09-17 09:17:07.488675946 +0000 UTC m=+0.468116527 container remove d7d4d8e93928ccbb19a7ea1489ca605e31bd2a3f907c509696bdc5e291d159d8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:17:07.557 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T09:17:07.557 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 systemd[1]: Stopped Ceph osd.4 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:17:07.557 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.4.service: Consumed 1.099s CPU time. 2024-09-17T09:17:07.947 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:07 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T09:17:07.948 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 systemd[1]: Starting Ceph osd.4 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:17:08.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:07 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T09:17:08.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:07 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-17T09:17:08.268 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 podman[81609]: 2024-09-17 09:17:07.849597445 +0000 UTC m=+0.015505633 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:08.269 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:07 smithi105 podman[81609]: 2024-09-17 09:17:07.95677147 +0000 UTC m=+0.122679652 container create e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T09:17:08.269 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 podman[81609]: 2024-09-17 09:17:08.151083082 +0000 UTC m=+0.316991271 container init e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:17:08.269 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 podman[81609]: 2024-09-17 09:17:08.154474315 +0000 UTC m=+0.320382503 container start e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:17:08.269 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 podman[81609]: 2024-09-17 09:17:08.267729537 +0000 UTC m=+0.433637735 container attach e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=reef-19e2a2b) 2024-09-17T09:17:08.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:17:08.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate[81624]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T09:17:08.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 bash[81609]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T09:17:08.530 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:08 smithi105 podman[81609]: 2024-09-17 09:17:08.494159005 +0000 UTC m=+0.660067185 container died e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b) 2024-09-17T09:17:08.959 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:08 smithi105 ceph-mon[70467]: pgmap v178: 161 pgs: 9 active+undersized, 17 peering, 11 stale+active+clean, 1 active+undersized+degraded, 123 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 9/999 objects degraded (0.901%) 2024-09-17T09:17:09.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:08 smithi027 ceph-mon[67349]: pgmap v178: 161 pgs: 9 active+undersized, 17 peering, 11 stale+active+clean, 1 active+undersized+degraded, 123 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 9/999 objects degraded (0.901%) 2024-09-17T09:17:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:08 smithi027 ceph-mon[71738]: pgmap v178: 161 pgs: 9 active+undersized, 17 peering, 11 stale+active+clean, 1 active+undersized+degraded, 123 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 9/999 objects degraded (0.901%) 2024-09-17T09:17:09.264 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 podman[81737]: 2024-09-17 09:17:09.071682059 +0000 UTC m=+0.571265615 container remove e99eeb854a313a49f00f6ffeee76b74eba934147822915ad4f9d253a993f946d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b) 2024-09-17T09:17:09.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 podman[81797]: 2024-09-17 09:17:09.26416881 +0000 UTC m=+0.102706854 container create aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T09:17:09.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 podman[81797]: 2024-09-17 09:17:09.177518445 +0000 UTC m=+0.016056504 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:09.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 podman[81797]: 2024-09-17 09:17:09.474874604 +0000 UTC m=+0.313412652 container init aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-17T09:17:09.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 podman[81797]: 2024-09-17 09:17:09.478221243 +0000 UTC m=+0.316759289 container start aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:17:09.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 ceph-osd[81815]: -- 172.21.15.105:0/2617174198 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e27929b1e0 con 0x55e279239c00 2024-09-17T09:17:10.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:09 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 9/999 objects degraded (0.901%), 1 pg degraded (PG_DEGRADED) 2024-09-17T09:17:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:09 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 9/999 objects degraded (0.901%), 1 pg degraded (PG_DEGRADED) 2024-09-17T09:17:10.019 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:09 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 9/999 objects degraded (0.901%), 1 pg degraded (PG_DEGRADED) 2024-09-17T09:17:10.020 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 bash[81797]: aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 2024-09-17T09:17:10.020 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:09 smithi105 systemd[1]: Started Ceph osd.4 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:17:10.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:17:10.018+0000 7fb0f3eb9640 -1 Falling back to public interface 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[67349]: pgmap v179: 161 pgs: 12 active+undersized, 17 peering, 9 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 47/999 objects degraded (4.705%) 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:10 smithi027 ceph-mon[71738]: pgmap v179: 161 pgs: 12 active+undersized, 17 peering, 9 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 47/999 objects degraded (4.705%) 2024-09-17T09:17:11.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:10 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:10 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:11.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:10 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:11.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:10 smithi105 ceph-mon[70467]: pgmap v179: 161 pgs: 12 active+undersized, 17 peering, 9 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 192 MiB used, 715 GiB / 715 GiB avail; 47/999 objects degraded (4.705%) 2024-09-17T09:17:12.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:11 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:17:12.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:11 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:17:12.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:11 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T09:17:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:12 smithi105 ceph-mon[70467]: pgmap v180: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:12 smithi105 ceph-mon[70467]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:12 smithi105 ceph-mon[70467]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.029 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:17:12.767+0000 7fb0f3eb9640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T09:17:13.029 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:17:12.803+0000 7fb0f3eb9640 -1 osd.4 97 log_to_monitors true 2024-09-17T09:17:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[71738]: pgmap v180: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[71738]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[71738]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[67349]: pgmap v180: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 196 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:13.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[67349]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:12 smithi027 ceph-mon[67349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:17:13.730 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:17:13.395+0000 7fb0e7c5b640 -1 osd.4 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:17:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-mon[70467]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:17:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-mon[70467]: osdmap e100: 8 total, 7 up, 8 in 2024-09-17T09:17:14.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-mon[70467]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-mon[70467]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:13 smithi105 ceph-mon[70467]: from='osd.4 ' entity='osd.4' 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[71738]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[71738]: osdmap e100: 8 total, 7 up, 8 in 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[71738]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[71738]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[71738]: from='osd.4 ' entity='osd.4' 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[67349]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:17:14.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[67349]: osdmap e100: 8 total, 7 up, 8 in 2024-09-17T09:17:14.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[67349]: from='osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[67349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:14.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:13 smithi027 ceph-mon[67349]: from='osd.4 ' entity='osd.4' 2024-09-17T09:17:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: pgmap v182: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:15.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294] boot 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: pgmap v182: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294] boot 2024-09-17T09:17:15.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T09:17:15.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:17:15.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:14 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: pgmap v182: 161 pgs: 43 active+undersized, 25 active+undersized+degraded, 93 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 163/999 objects degraded (16.316%) 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: osd.4 [v2:172.21.15.105:6800/1736582294,v1:172.21.15.105:6801/1736582294] boot 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T09:17:15.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:17:15.265 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:15.265 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:14 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:16.609 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:16 smithi105 ceph-mon[70467]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T09:17:16.610 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:16 smithi105 ceph-mon[70467]: pgmap v185: 161 pgs: 26 peering, 24 active+undersized, 15 active+undersized+degraded, 96 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 79/999 objects degraded (7.908%) 2024-09-17T09:17:16.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:16 smithi027 ceph-mon[67349]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T09:17:16.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:16 smithi027 ceph-mon[67349]: pgmap v185: 161 pgs: 26 peering, 24 active+undersized, 15 active+undersized+degraded, 96 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 79/999 objects degraded (7.908%) 2024-09-17T09:17:16.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:16 smithi027 ceph-mon[71738]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T09:17:16.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:16 smithi027 ceph-mon[71738]: pgmap v185: 161 pgs: 26 peering, 24 active+undersized, 15 active+undersized+degraded, 96 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 79/999 objects degraded (7.908%) 2024-09-17T09:17:17.682 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:17 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 79/999 objects degraded (7.908%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:17.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:17 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 79/999 objects degraded (7.908%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:17.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:17 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 79/999 objects degraded (7.908%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:18.508 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:17:18.178+0000 7f5acd221640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T09:17:18.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: pgmap v186: 161 pgs: 26 peering, 15 active+undersized, 12 active+undersized+degraded, 108 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 57/999 objects degraded (5.706%) 2024-09-17T09:17:18.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: pgmap v186: 161 pgs: 26 peering, 15 active+undersized, 12 active+undersized+degraded, 108 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 57/999 objects degraded (5.706%) 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: pgmap v186: 161 pgs: 26 peering, 15 active+undersized, 12 active+undersized+degraded, 108 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 57/999 objects degraded (5.706%) 2024-09-17T09:17:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:18.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: pgmap v187: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: pgmap v188: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T09:17:19.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: pgmap v187: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: pgmap v188: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T09:17:19.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: pgmap v187: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: pgmap v188: 161 pgs: 26 peering, 1 active+undersized, 3 active+undersized+degraded, 131 active+clean; 604 KiB data, 216 MiB used, 715 GiB / 715 GiB avail; 17/999 objects degraded (1.702%); 0 B/s, 0 objects/s recovering 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T09:17:19.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:20.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:20 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/999 objects degraded (1.702%), 3 pgs degraded) 2024-09-17T09:17:20.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:20 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/999 objects degraded (1.702%), 3 pgs degraded) 2024-09-17T09:17:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:20 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/999 objects degraded (1.702%), 3 pgs degraded) 2024-09-17T09:17:21.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:21 smithi027 ceph-mon[67349]: pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:21.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:21 smithi027 ceph-mon[71738]: pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:21.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:21 smithi105 ceph-mon[70467]: pgmap v189: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:23.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:23 smithi027 ceph-mon[67349]: pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:23.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:23 smithi027 ceph-mon[71738]: pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:23.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:23 smithi105 ceph-mon[70467]: pgmap v190: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:25.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:25 smithi027 ceph-mon[67349]: pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:25.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:25 smithi027 ceph-mon[71738]: pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:25.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:25 smithi105 ceph-mon[70467]: pgmap v191: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:27.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:27 smithi027 ceph-mon[67349]: pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:27.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:27 smithi027 ceph-mon[71738]: pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:27 smithi105 ceph-mon[70467]: pgmap v192: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-17T09:17:30.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:29 smithi027 ceph-mon[67349]: pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:30.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:29 smithi027 ceph-mon[71738]: pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:30.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:30.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:29 smithi105 ceph-mon[70467]: pgmap v193: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:30.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:32.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:31 smithi027 ceph-mon[67349]: pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:32.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:31 smithi027 ceph-mon[71738]: pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:31 smithi105 ceph-mon[70467]: pgmap v194: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:33.133 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:17:33.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:33 smithi105 ceph-mon[70467]: pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:33.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:33 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:33.792 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:17:33.792 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 52s ago 28m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:17:33.792 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (6m) 19s ago 28m 89.4M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 19s ago 28m - - 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (8m) 19s ago 32m 438M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (18m) 52s ago 33m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (4m) 52s ago 34m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (5m) 19s ago 32m 54.4M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (4m) 52s ago 33m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (7m) 52s ago 29m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (7m) 19s ago 29m 16.6M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (3m) 52s ago 32m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (2m) 52s ago 32m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (112s) 52s ago 31m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:17:33.793 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (56s) 52s ago 31m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (24s) 19s ago 31m 11.8M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (30m) 19s ago 30m 50.4M 4096M 16.2.0 24ecd6d5f14c 3cca2c3ba8c4 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (30m) 19s ago 30m 53.3M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (30m) 19s ago 30m 57.2M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (7m) 19s ago 29m 53.5M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (28m) 52s ago 28m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:17:33.794 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (28m) 19s ago 28m 45.3M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:17:33.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:33 smithi027 ceph-mon[67349]: pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:33.794 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:33 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:33.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:33 smithi027 ceph-mon[71738]: pgmap v195: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:33.795 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:33 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:17:34.195 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3, 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 5 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 5, 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 10 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:17:34.196 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "5/8 daemons upgraded", 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:17:34.535 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: Upgrade: osd.5 is safe to restart 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: from='client.54504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[67349]: from='client.54516 172.21.15.27:0/1231960221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: Upgrade: osd.5 is safe to restart 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: from='client.54504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:34 smithi027 ceph-mon[71738]: from='client.54516 172.21.15.27:0/1231960221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: from='client.54498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: Upgrade: osd.5 is safe to restart 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: from='client.54504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: from='client.64450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:35.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:34 smithi105 ceph-mon[70467]: from='client.54516 172.21.15.27:0/1231960221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: Upgrade: Updating osd.5 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:17:36.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[67349]: Deploying daemon osd.5 on smithi105 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: Upgrade: Updating osd.5 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:36.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:35 smithi027 ceph-mon[71738]: Deploying daemon osd.5 on smithi105 2024-09-17T09:17:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: pgmap v196: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: from='client.64462 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:17:36.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: Upgrade: Updating osd.5 2024-09-17T09:17:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:17:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:35 smithi105 ceph-mon[70467]: Deploying daemon osd.5 on smithi105 2024-09-17T09:17:37.695 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:37 smithi105 systemd[1]: Stopping Ceph osd.5 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:17:38.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:37 smithi027 ceph-mon[67349]: pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:37 smithi027 ceph-mon[71738]: pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:38.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:37 smithi105 ceph-mon[70467]: pgmap v197: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:38.020 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T09:17:37.748+0000 7f566b1a3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:17:38.020 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T09:17:37.748+0000 7f566b1a3700 -1 osd.5 102 *** Got signal Terminated *** 2024-09-17T09:17:38.020 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5[28860]: debug 2024-09-17T09:17:37.748+0000 7f566b1a3700 -1 osd.5 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:17:38.020 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:37 smithi105 podman[86183]: 2024-09-17 09:17:37.790339686 +0000 UTC m=+0.169646399 container died 3cca2c3ba8c4d22c4aceadcd5cb10144cf9d7b54f3dd4be01e6137f8108b89d5 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.build-date=20201204, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Dimitri Savineau ) 2024-09-17T09:17:38.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:38 smithi105 podman[86183]: 2024-09-17 09:17:38.266686167 +0000 UTC m=+0.645992880 container cleanup 3cca2c3ba8c4d22c4aceadcd5cb10144cf9d7b54f3dd4be01e6137f8108b89d5 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD, org.label-schema.build-date=20201204, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Dimitri Savineau , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Base Image, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557) 2024-09-17T09:17:38.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:38 smithi105 bash[86183]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5 2024-09-17T09:17:38.778 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:38 smithi105 podman[86196]: 2024-09-17 09:17:38.5263988 +0000 UTC m=+0.739966421 container remove 3cca2c3ba8c4d22c4aceadcd5cb10144cf9d7b54f3dd4be01e6137f8108b89d5 (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, ceph=True, org.label-schema.build-date=20201204, org.label-schema.name=CentOS Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.license=GPLv2, maintainer=Dimitri Savineau ) 2024-09-17T09:17:38.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 failed (root=default,host=smithi105) (connection refused reported by osd.1) 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:38.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:38.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:38.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:38.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:38 smithi105 ceph-mon[70467]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 failed (root=default,host=smithi105) (connection refused reported by osd.1) 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[67349]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 failed (root=default,host=smithi105) (connection refused reported by osd.1) 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.012 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.0 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.1 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.3 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.2 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.7 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.6 2024-09-17T09:17:39.013 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:38 smithi027 ceph-mon[71738]: osd.5 reported immediately failed by osd.4 2024-09-17T09:17:39.146 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:38.91823736 +0000 UTC m=+0.016063088 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:39.146 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:39.035891157 +0000 UTC m=+0.133716879 container create 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:17:39.469 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:39.273467218 +0000 UTC m=+0.371292941 container init 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b) 2024-09-17T09:17:39.469 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:39.276981406 +0000 UTC m=+0.374807129 container start 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T09:17:39.469 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:39.371599423 +0000 UTC m=+0.469425158 container attach 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T09:17:39.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:39 smithi105 ceph-mon[70467]: pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:39.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:39 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:39.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:39 smithi105 ceph-mon[70467]: osdmap e103: 8 total, 7 up, 8 in 2024-09-17T09:17:39.779 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:39 smithi105 podman[86311]: 2024-09-17 09:17:39.469138101 +0000 UTC m=+0.566963831 container died 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T09:17:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[67349]: pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:40.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[67349]: osdmap e103: 8 total, 7 up, 8 in 2024-09-17T09:17:40.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[71738]: pgmap v198: 161 pgs: 161 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:40.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:17:40.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:39 smithi027 ceph-mon[71738]: osdmap e103: 8 total, 7 up, 8 in 2024-09-17T09:17:40.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86334]: 2024-09-17 09:17:40.10946497 +0000 UTC m=+0.630102490 container remove 0ff34af56173ffa763bfd1e3fb70318613ee91ed3f34972a033d89cdfdd52ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux ) 2024-09-17T09:17:40.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T09:17:40.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 systemd[1]: Stopped Ceph osd.5 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:17:40.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.5.service: Consumed 1.150s CPU time. 2024-09-17T09:17:40.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 systemd[1]: Starting Ceph osd.5 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:17:40.797 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:40 smithi105 ceph-mon[70467]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T09:17:40.797 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86434]: 2024-09-17 09:17:40.43599855 +0000 UTC m=+0.015823518 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:40.797 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86434]: 2024-09-17 09:17:40.556006923 +0000 UTC m=+0.135831892 container create b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:17:41.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:40 smithi027 ceph-mon[67349]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T09:17:41.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:40 smithi027 ceph-mon[71738]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T09:17:41.095 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86434]: 2024-09-17 09:17:40.796549979 +0000 UTC m=+0.376374945 container init b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T09:17:41.096 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86434]: 2024-09-17 09:17:40.799983017 +0000 UTC m=+0.379807986 container start b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:17:41.096 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:40 smithi105 podman[86434]: 2024-09-17 09:17:40.879790843 +0000 UTC m=+0.459615817 container attach b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=reef-19e2a2b) 2024-09-17T09:17:41.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:17:41.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate[86450]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T09:17:41.529 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 bash[86434]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T09:17:41.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 podman[86434]: 2024-09-17 09:17:41.142007981 +0000 UTC m=+0.721832948 container died b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T09:17:41.972 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-mon[70467]: pgmap v201: 161 pgs: 10 active+undersized, 6 peering, 15 stale+active+clean, 6 active+undersized+degraded, 124 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 36/999 objects degraded (3.604%) 2024-09-17T09:17:41.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:17:41.973 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:41 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 36/999 objects degraded (3.604%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:41.973 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 podman[86573]: 2024-09-17 09:17:41.767015632 +0000 UTC m=+0.619116071 container remove b5089e0bba72d236403c4665e533a75842c5c2f35a411c84920a2781d9eb284f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0) 2024-09-17T09:17:42.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[67349]: pgmap v201: 161 pgs: 10 active+undersized, 6 peering, 15 stale+active+clean, 6 active+undersized+degraded, 124 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 36/999 objects degraded (3.604%) 2024-09-17T09:17:42.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:17:42.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 36/999 objects degraded (3.604%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:42.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[71738]: pgmap v201: 161 pgs: 10 active+undersized, 6 peering, 15 stale+active+clean, 6 active+undersized+degraded, 124 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 36/999 objects degraded (3.604%) 2024-09-17T09:17:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:17:42.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:41 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 36/999 objects degraded (3.604%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:42.238 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 podman[86632]: 2024-09-17 09:17:41.874407375 +0000 UTC m=+0.015632330 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:17:42.238 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:41 smithi105 podman[86632]: 2024-09-17 09:17:41.976169726 +0000 UTC m=+0.117394658 container create c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:17:42.238 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 podman[86632]: 2024-09-17 09:17:42.2376955 +0000 UTC m=+0.378920472 container init c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T09:17:42.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 podman[86632]: 2024-09-17 09:17:42.241036716 +0000 UTC m=+0.382261647 container start c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:17:42.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 ceph-osd[86651]: -- 172.21.15.105:0/3159827053 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56186c36b1e0 con 0x56186d0ec000 2024-09-17T09:17:42.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 bash[86632]: c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d 2024-09-17T09:17:42.528 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 systemd[1]: Started Ceph osd.5 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:17:43.080 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:17:42.778+0000 7f6c2df01640 -1 Falling back to public interface 2024-09-17T09:17:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[67349]: pgmap v202: 161 pgs: 26 active+undersized, 6 peering, 9 stale+active+clean, 8 active+undersized+degraded, 112 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 51/999 objects degraded (5.105%) 2024-09-17T09:17:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:43.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[71738]: pgmap v202: 161 pgs: 26 active+undersized, 6 peering, 9 stale+active+clean, 8 active+undersized+degraded, 112 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 51/999 objects degraded (5.105%) 2024-09-17T09:17:43.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:43 smithi105 ceph-mon[70467]: pgmap v202: 161 pgs: 26 active+undersized, 6 peering, 9 stale+active+clean, 8 active+undersized+degraded, 112 active+clean; 604 KiB data, 220 MiB used, 715 GiB / 715 GiB avail; 51/999 objects degraded (5.105%) 2024-09-17T09:17:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:45.675 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:45 smithi105 ceph-mon[70467]: pgmap v203: 161 pgs: 37 active+undersized, 6 peering, 16 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 99/999 objects degraded (9.910%) 2024-09-17T09:17:45.676 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:17:45.499+0000 7f6c2df01640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T09:17:45.676 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:45 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:17:45.523+0000 7f6c2df01640 -1 osd.5 102 log_to_monitors true 2024-09-17T09:17:45.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:45 smithi027 ceph-mon[71738]: pgmap v203: 161 pgs: 37 active+undersized, 6 peering, 16 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 99/999 objects degraded (9.910%) 2024-09-17T09:17:45.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:45 smithi027 ceph-mon[67349]: pgmap v203: 161 pgs: 37 active+undersized, 6 peering, 16 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 99/999 objects degraded (9.910%) 2024-09-17T09:17:46.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:46 smithi105 ceph-mon[70467]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:46.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:46 smithi105 ceph-mon[70467]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:46.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:46 smithi027 ceph-mon[67349]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:46.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:46 smithi027 ceph-mon[67349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:46.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:46 smithi027 ceph-mon[71738]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:46.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:46 smithi027 ceph-mon[71738]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:17:47.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:17:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:17:46.848+0000 7f6c21ca3640 -1 osd.5 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:17:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: pgmap v204: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:17:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 115/999 objects degraded (11.512%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T09:17:47.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='osd.5 ' entity='osd.5' 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: pgmap v204: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 115/999 objects degraded (11.512%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:47.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='osd.5 ' entity='osd.5' 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:47.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: pgmap v204: 161 pgs: 40 active+undersized, 19 active+undersized+degraded, 102 active+clean; 604 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 115/999 objects degraded (11.512%) 2024-09-17T09:17:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 115/999 objects degraded (11.512%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-17T09:17:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:17:47.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T09:17:47.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:17:47.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='osd.5 ' entity='osd.5' 2024-09-17T09:17:47.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:47.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894] boot 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T09:17:48.871 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:48 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:17:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:49.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894] boot 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: osd.5 [v2:172.21.15.105:6808/3092861894,v1:172.21.15.105:6809/3092861894] boot 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T09:17:49.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:48 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:17:50.203 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:49 smithi105 ceph-mon[70467]: pgmap v207: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 604 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 114/999 objects degraded (11.411%) 2024-09-17T09:17:50.203 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:49 smithi105 ceph-mon[70467]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T09:17:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:49 smithi027 ceph-mon[67349]: pgmap v207: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 604 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 114/999 objects degraded (11.411%) 2024-09-17T09:17:50.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:49 smithi027 ceph-mon[67349]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T09:17:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:49 smithi027 ceph-mon[71738]: pgmap v207: 161 pgs: 5 peering, 36 active+undersized, 18 active+undersized+degraded, 102 active+clean; 604 KiB data, 240 MiB used, 715 GiB / 715 GiB avail; 114/999 objects degraded (11.411%) 2024-09-17T09:17:50.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:49 smithi027 ceph-mon[71738]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T09:17:51.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:17:50 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:17:50.680+0000 7f5acd221640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T09:17:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: pgmap v209: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: pgmap v210: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: pgmap v211: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[67349]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: pgmap v209: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:52.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: pgmap v210: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: pgmap v211: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.011 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:51 smithi027 ceph-mon[71738]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T09:17:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: pgmap v209: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:17:52.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: pgmap v210: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: pgmap v211: 161 pgs: 5 peering, 28 active+undersized, 11 active+undersized+degraded, 117 active+clean; 604 KiB data, 241 MiB used, 715 GiB / 715 GiB avail; 74/999 objects degraded (7.407%) 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:17:52.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:51 smithi105 ceph-mon[70467]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T09:17:53.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:52 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:53.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:52 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:53.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:52 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:17:54.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:53 smithi027 ceph-mon[67349]: pgmap v212: 161 pgs: 5 peering, 156 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:54.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:53 smithi027 ceph-mon[71738]: pgmap v212: 161 pgs: 5 peering, 156 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:54.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:53 smithi105 ceph-mon[70467]: pgmap v212: 161 pgs: 5 peering, 156 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:54 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded) 2024-09-17T09:17:55.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:54 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded) 2024-09-17T09:17:55.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:54 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/999 objects degraded (7.407%), 11 pgs degraded) 2024-09-17T09:17:55.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:17:55.909 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:55 smithi027 ceph-mon[67349]: pgmap v213: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:55.910 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:55 smithi027 ceph-mon[71738]: pgmap v213: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:56.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:55 smithi105 ceph-mon[70467]: pgmap v213: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:57.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:57 smithi027 ceph-mon[67349]: pgmap v214: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:57.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:57 smithi027 ceph-mon[71738]: pgmap v214: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:17:58.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:57 smithi105 ceph-mon[70467]: pgmap v214: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:00.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:17:59 smithi027 ceph-mon[67349]: pgmap v215: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:00.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:17:59 smithi027 ceph-mon[71738]: pgmap v215: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:00.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:17:59 smithi105 ceph-mon[70467]: pgmap v215: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:02.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:01 smithi027 ceph-mon[67349]: pgmap v216: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:02.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:01 smithi027 ceph-mon[71738]: pgmap v216: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:02.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:01 smithi105 ceph-mon[70467]: pgmap v216: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:04.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:03 smithi027 ceph-mon[71738]: pgmap v217: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:04.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:03 smithi027 ceph-mon[67349]: pgmap v217: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:04.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:03 smithi105 ceph-mon[70467]: pgmap v217: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:04.903 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:18:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[67349]: pgmap v218: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:05.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[67349]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:05.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[71738]: pgmap v218: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:05.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:05 smithi027 ceph-mon[71738]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:05 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:05 smithi105 ceph-mon[70467]: pgmap v218: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:05 smithi105 ceph-mon[70467]: from='client.54531 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (7m) 83s ago 29m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (7m) 18s ago 28m 89.4M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 18s ago 28m - - 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (8m) 18s ago 33m 438M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (18m) 83s ago 34m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (5m) 83s ago 34m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (6m) 18s ago 33m 55.1M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:18:05.563 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 83s ago 33m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 83s ago 29m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (8m) 18s ago 29m 16.6M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (3m) 83s ago 32m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (3m) 83s ago 32m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (2m) 83s ago 32m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (87s) 83s ago 31m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (56s) 18s ago 31m 51.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (23s) 18s ago 31m 12.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (31m) 18s ago 31m 53.9M 4096M 16.2.0 24ecd6d5f14c c7bbd06e6775 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (30m) 18s ago 30m 57.5M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (7m) 18s ago 30m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (28m) 83s ago 28m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:18:05.564 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (28m) 18s ago 28m 45.4M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:18:05.960 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2, 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 6 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 4, 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 11 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:18:05.961 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: Upgrade: osd.6 is safe to restart 2024-09-17T09:18:06.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[67349]: from='client.54549 172.21.15.27:0/4047670050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: Upgrade: osd.6 is safe to restart 2024-09-17T09:18:06.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:06 smithi027 ceph-mon[71738]: from='client.54549 172.21.15.27:0/4047670050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: from='client.54537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: from='client.64483 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:18:06.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: Upgrade: osd.6 is safe to restart 2024-09-17T09:18:06.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:06 smithi105 ceph-mon[70467]: from='client.54549 172.21.15.27:0/4047670050' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:18:06.298 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "6/8 daemons upgraded", 2024-09-17T09:18:06.299 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:18:06.299 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:18:06.299 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:18:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:07 smithi105 ceph-mon[70467]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:07 smithi105 ceph-mon[70467]: pgmap v219: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:07.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:07 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:07.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:07 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:18:07.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:07 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[67349]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[67349]: pgmap v219: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:18:07.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:07.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[71738]: from='client.64495 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:07.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[71738]: pgmap v219: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:07.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:07.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:18:07.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:07 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:08.760 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:08 smithi105 ceph-mon[70467]: Upgrade: Updating osd.6 2024-09-17T09:18:08.760 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:08 smithi105 ceph-mon[70467]: Deploying daemon osd.6 on smithi105 2024-09-17T09:18:09.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:08 smithi027 ceph-mon[67349]: Upgrade: Updating osd.6 2024-09-17T09:18:09.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:08 smithi027 ceph-mon[67349]: Deploying daemon osd.6 on smithi105 2024-09-17T09:18:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:08 smithi027 ceph-mon[71738]: Upgrade: Updating osd.6 2024-09-17T09:18:09.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:08 smithi027 ceph-mon[71738]: Deploying daemon osd.6 on smithi105 2024-09-17T09:18:09.878 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:09 smithi105 ceph-mon[70467]: pgmap v220: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:10.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:09 smithi027 ceph-mon[67349]: pgmap v220: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:10.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:09 smithi027 ceph-mon[71738]: pgmap v220: 161 pgs: 161 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:10.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:09 smithi105 systemd[1]: Stopping Ceph osd.6 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:18:10.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T09:18:10.018+0000 7ff72b5bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:18:10.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T09:18:10.018+0000 7ff72b5bc700 -1 osd.6 107 *** Got signal Terminated *** 2024-09-17T09:18:10.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6[31231]: debug 2024-09-17T09:18:10.018+0000 7ff72b5bc700 -1 osd.6 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:18:10.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 podman[90759]: 2024-09-17 09:18:10.049970222 +0000 UTC m=+0.132486129 container died c7bbd06e6775512e991f148b158b77179d4272b3faba67ce455c6a411df5e8dc (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6, org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.name=CentOS Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T09:18:10.547 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 podman[90759]: 2024-09-17 09:18:10.418629108 +0000 UTC m=+0.501145015 container cleanup c7bbd06e6775512e991f148b158b77179d4272b3faba67ce455c6a411df5e8dc (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Dimitri Savineau , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=-16.2.0, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, RELEASE=HEAD, org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T09:18:10.547 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 bash[90759]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 failed (root=default,host=smithi105) (connection refused reported by osd.5) 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[71738]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 failed (root=default,host=smithi105) (connection refused reported by osd.5) 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.011 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:10 smithi027 ceph-mon[67349]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:10 smithi105 podman[90772]: 2024-09-17 09:18:10.670068912 +0000 UTC m=+0.614659124 container remove c7bbd06e6775512e991f148b158b77179d4272b3faba67ce455c6a411df5e8dc (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.6, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, maintainer=Dimitri Savineau , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Base Image, org.label-schema.build-date=20201204, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T09:18:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 failed (root=default,host=smithi105) (connection refused reported by osd.5) 2024-09-17T09:18:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.5 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.0 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.1 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.3 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.2 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.4 2024-09-17T09:18:11.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:10 smithi105 ceph-mon[70467]: osd.6 reported immediately failed by osd.7 2024-09-17T09:18:11.399 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.031955204 +0000 UTC m=+0.016045177 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:11.399 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.129633823 +0000 UTC m=+0.113723784 container create 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T09:18:11.400 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.333306381 +0000 UTC m=+0.317396335 container init 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T09:18:11.400 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.336570804 +0000 UTC m=+0.320660766 container start 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:18:11.400 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.398804002 +0000 UTC m=+0.382893966 container attach 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T09:18:11.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:11 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:11.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:11 smithi105 ceph-mon[70467]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T09:18:11.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:11 smithi105 ceph-mon[70467]: pgmap v222: 161 pgs: 13 stale+active+clean, 148 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:11.696 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:11 smithi105 podman[90880]: 2024-09-17 09:18:11.538582026 +0000 UTC m=+0.522671995 container died 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[67349]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[67349]: pgmap v222: 161 pgs: 13 stale+active+clean, 148 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[71738]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T09:18:12.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:11 smithi027 ceph-mon[71738]: pgmap v222: 161 pgs: 13 stale+active+clean, 148 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:12.457 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[90904]: 2024-09-17 09:18:12.119607588 +0000 UTC m=+0.574076197 container remove 003c1c723505390b7d0e48b74f543b7c36db7fb7049d22cc4b348fb2a9356e93 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:18:12.457 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T09:18:12.457 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 systemd[1]: Stopped Ceph osd.6 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:18:12.457 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.6.service: Consumed 1.092s CPU time. 2024-09-17T09:18:12.457 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 systemd[1]: Starting Ceph osd.6 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:18:12.765 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:12 smithi105 ceph-mon[70467]: osdmap e109: 8 total, 7 up, 8 in 2024-09-17T09:18:12.766 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[91012]: 2024-09-17 09:18:12.545948669 +0000 UTC m=+0.107609763 container create a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:18:12.766 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[91012]: 2024-09-17 09:18:12.455475698 +0000 UTC m=+0.017136791 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:12.766 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[91012]: 2024-09-17 09:18:12.765136902 +0000 UTC m=+0.326797999 container init a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T09:18:13.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:12 smithi027 ceph-mon[67349]: osdmap e109: 8 total, 7 up, 8 in 2024-09-17T09:18:13.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:12 smithi027 ceph-mon[71738]: osdmap e109: 8 total, 7 up, 8 in 2024-09-17T09:18:13.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[91012]: 2024-09-17 09:18:12.768612442 +0000 UTC m=+0.330273536 container start a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T09:18:13.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:12 smithi105 podman[91012]: 2024-09-17 09:18:12.848417991 +0000 UTC m=+0.410079094 container attach a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T09:18:13.529 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:18:13.529 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:18:13.529 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:18:13.529 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:18:13.529 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate[91028]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 bash[91012]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T09:18:13.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 podman[91012]: 2024-09-17 09:18:13.113740808 +0000 UTC m=+0.675401902 container died a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T09:18:13.882 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-mon[70467]: pgmap v224: 161 pgs: 7 active+undersized, 14 peering, 9 stale+active+clean, 4 active+undersized+degraded, 127 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 24/999 objects degraded (2.402%) 2024-09-17T09:18:13.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T09:18:13.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:13 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 24/999 objects degraded (2.402%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:13.883 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 podman[91140]: 2024-09-17 09:18:13.677018743 +0000 UTC m=+0.557424373 container remove a8a8f5bc3291ad9505f49f012cc70908f38805f399dd93a5e7ff6dbf5c2610e0 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T09:18:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[67349]: pgmap v224: 161 pgs: 7 active+undersized, 14 peering, 9 stale+active+clean, 4 active+undersized+degraded, 127 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 24/999 objects degraded (2.402%) 2024-09-17T09:18:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T09:18:14.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 24/999 objects degraded (2.402%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:14.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[71738]: pgmap v224: 161 pgs: 7 active+undersized, 14 peering, 9 stale+active+clean, 4 active+undersized+degraded, 127 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 24/999 objects degraded (2.402%) 2024-09-17T09:18:14.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T09:18:14.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:13 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 24/999 objects degraded (2.402%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:14.223 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 podman[91197]: 2024-09-17 09:18:13.785094708 +0000 UTC m=+0.015833172 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:14.223 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:13 smithi105 podman[91197]: 2024-09-17 09:18:13.886314012 +0000 UTC m=+0.117052503 container create bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b) 2024-09-17T09:18:14.223 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 podman[91197]: 2024-09-17 09:18:14.222251379 +0000 UTC m=+0.452989833 container init bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-17T09:18:14.528 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 podman[91197]: 2024-09-17 09:18:14.225580231 +0000 UTC m=+0.456318681 container start bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T09:18:14.528 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 ceph-osd[91215]: -- 172.21.15.105:0/1651083735 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559fc22211e0 con 0x559fc2fa0400 2024-09-17T09:18:14.528 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 bash[91197]: bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b 2024-09-17T09:18:14.528 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 systemd[1]: Started Ceph osd.6 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:18:15.017 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:18:14.764+0000 7fd62dd6c640 -1 Falling back to public interface 2024-09-17T09:18:15.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:15.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[67349]: pgmap v225: 161 pgs: 20 active+undersized, 14 peering, 1 stale+active+clean, 9 active+undersized+degraded, 117 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 67/999 objects degraded (6.707%) 2024-09-17T09:18:15.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:15.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:15 smithi027 ceph-mon[71738]: pgmap v225: 161 pgs: 20 active+undersized, 14 peering, 1 stale+active+clean, 9 active+undersized+degraded, 117 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 67/999 objects degraded (6.707%) 2024-09-17T09:18:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:15 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:15.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:15 smithi105 ceph-mon[70467]: pgmap v225: 161 pgs: 20 active+undersized, 14 peering, 1 stale+active+clean, 9 active+undersized+degraded, 117 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 67/999 objects degraded (6.707%) 2024-09-17T09:18:17.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:16 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-17T09:18:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:16 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-17T09:18:17.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:16 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-17T09:18:17.778 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:18:17.498+0000 7fd62dd6c640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T09:18:17.778 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:18:17.536+0000 7fd62dd6c640 -1 osd.6 107 log_to_monitors true 2024-09-17T09:18:17.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:17 smithi105 ceph-mon[70467]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:17.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:17 smithi105 ceph-mon[70467]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:17.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:17 smithi105 ceph-mon[70467]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[71738]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[71738]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:18.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[71738]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:18.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[67349]: pgmap v226: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 245 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:18.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[67349]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:18.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:17 smithi027 ceph-mon[67349]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:18:18.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:18:18.134+0000 7fd621b0e640 -1 osd.6 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[71738]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[71738]: osdmap e110: 8 total, 7 up, 8 in 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[71738]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[71738]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[71738]: from='osd.6 ' entity='osd.6' 2024-09-17T09:18:19.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[67349]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:18:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[67349]: osdmap e110: 8 total, 7 up, 8 in 2024-09-17T09:18:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[67349]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[67349]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:18 smithi027 ceph-mon[67349]: from='osd.6 ' entity='osd.6' 2024-09-17T09:18:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-mon[70467]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:18:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-mon[70467]: osdmap e110: 8 total, 7 up, 8 in 2024-09-17T09:18:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-mon[70467]: from='osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-mon[70467]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:18 smithi105 ceph-mon[70467]: from='osd.6 ' entity='osd.6' 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: pgmap v228: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111] boot 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:19 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: pgmap v228: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111] boot 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: pgmap v228: 161 pgs: 35 active+undersized, 18 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 122/999 objects degraded (12.212%) 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: osd.6 [v2:172.21.15.105:6816/1573643111,v1:172.21.15.105:6817/1573643111] boot 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:20.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:19 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:21.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:21 smithi027 ceph-mon[67349]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T09:18:21.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:21 smithi027 ceph-mon[67349]: pgmap v231: 161 pgs: 10 peering, 28 active+undersized, 15 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 100/999 objects degraded (10.010%) 2024-09-17T09:18:21.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:21 smithi027 ceph-mon[71738]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T09:18:21.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:21 smithi027 ceph-mon[71738]: pgmap v231: 161 pgs: 10 peering, 28 active+undersized, 15 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 100/999 objects degraded (10.010%) 2024-09-17T09:18:21.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:21 smithi105 ceph-mon[70467]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T09:18:21.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:21 smithi105 ceph-mon[70467]: pgmap v231: 161 pgs: 10 peering, 28 active+undersized, 15 active+undersized+degraded, 108 active+clean; 604 KiB data, 268 MiB used, 715 GiB / 715 GiB avail; 100/999 objects degraded (10.010%) 2024-09-17T09:18:22.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:22 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 100/999 objects degraded (10.010%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:22.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:22 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 100/999 objects degraded (10.010%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:22.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:22 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 100/999 objects degraded (10.010%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:22.508 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:18:22 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:18:22.395+0000 7f5acd221640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-09-17T09:18:23.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: pgmap v232: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: pgmap v233: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[67349]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:23.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: pgmap v232: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: pgmap v233: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:23 smithi027 ceph-mon[71738]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-09-17T09:18:23.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:23.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:23.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: pgmap v232: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: pgmap v233: 161 pgs: 10 peering, 15 active+undersized, 7 active+undersized+degraded, 129 active+clean; 604 KiB data, 269 MiB used, 715 GiB / 715 GiB avail; 45/999 objects degraded (4.505%) 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:23.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:23.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:23 smithi105 ceph-mon[70467]: Upgrade: unsafe to stop osd(s) at this time (16 PGs are or would become offline) 2024-09-17T09:18:25.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:25 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:25.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:25 smithi027 ceph-mon[67349]: pgmap v234: 161 pgs: 10 peering, 151 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:25.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:25 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:25.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:25 smithi027 ceph-mon[71738]: pgmap v234: 161 pgs: 10 peering, 151 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:25.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:25 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:25.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:25 smithi105 ceph-mon[70467]: pgmap v234: 161 pgs: 10 peering, 151 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:26.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:26 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/999 objects degraded (4.505%), 7 pgs degraded) 2024-09-17T09:18:26.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:26 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/999 objects degraded (4.505%), 7 pgs degraded) 2024-09-17T09:18:26.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:26 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/999 objects degraded (4.505%), 7 pgs degraded) 2024-09-17T09:18:27.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:27 smithi027 ceph-mon[67349]: pgmap v235: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:27.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:27 smithi027 ceph-mon[71738]: pgmap v235: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:27.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:27 smithi105 ceph-mon[70467]: pgmap v235: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:29.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:29 smithi027 ceph-mon[67349]: pgmap v236: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:29.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:29 smithi027 ceph-mon[71738]: pgmap v236: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:29.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:29 smithi105 ceph-mon[70467]: pgmap v236: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:31.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:31 smithi027 ceph-mon[67349]: pgmap v237: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:31.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:31 smithi027 ceph-mon[71738]: pgmap v237: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:31.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:31 smithi105 ceph-mon[70467]: pgmap v237: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:33.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:33 smithi027 ceph-mon[67349]: pgmap v238: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:33.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:33 smithi027 ceph-mon[71738]: pgmap v238: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:33.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:33 smithi105 ceph-mon[70467]: pgmap v238: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:35 smithi027 ceph-mon[67349]: pgmap v239: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:35.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:35 smithi027 ceph-mon[71738]: pgmap v239: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:35 smithi105 ceph-mon[70467]: pgmap v239: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:36.668 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 115s ago 29m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (7m) 18s ago 29m 88.6M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 18s ago 29m - - 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (9m) 18s ago 33m 438M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (19m) 115s ago 35m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (5m) 115s ago 35m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (6m) 18s ago 33m 55.6M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (5m) 115s ago 34m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:18:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (8m) 115s ago 30m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (8m) 18s ago 30m 16.8M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (4m) 115s ago 33m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (3m) 115s ago 33m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (2m) 115s ago 32m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (119s) 115s ago 32m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (88s) 18s ago 32m 53.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (55s) 18s ago 31m 51.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (23s) 18s ago 31m 14.3M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (31m) 18s ago 31m 58.7M 4096M 16.2.0 24ecd6d5f14c fda5accbb1e3 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (8m) 18s ago 30m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (29m) 115s ago 29m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:18:37.334 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (29m) 18s ago 29m 45.5M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:18:37.733 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:18:37.733 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:18:37.733 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 1, 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 7 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:18:37.734 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:18:37.735 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 3, 2024-09-17T09:18:37.735 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 12 2024-09-17T09:18:37.735 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:18:37.735 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:18:37.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[67349]: pgmap v240: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:37.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[67349]: from='client.54564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:37.735 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[67349]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:37.735 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[71738]: pgmap v240: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:37.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[71738]: from='client.54564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:37.736 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:37 smithi027 ceph-mon[71738]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:37.741 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:37 smithi105 ceph-mon[70467]: pgmap v240: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:37.741 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:37 smithi105 ceph-mon[70467]: from='client.54564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:37.741 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:37 smithi105 ceph-mon[70467]: from='client.64510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:38.068 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:18:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:18:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:18:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-09-17T09:18:38.068 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:18:38.069 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "7/8 daemons upgraded", 2024-09-17T09:18:38.069 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:18:38.069 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:18:38.069 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:18:38.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:38 smithi105 ceph-mon[70467]: from='client.54576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:38.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:38 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:38 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:38 smithi105 ceph-mon[70467]: Upgrade: osd.7 is safe to restart 2024-09-17T09:18:38.745 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:38 smithi105 ceph-mon[70467]: from='client.64522 172.21.15.27:0/2629881563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[67349]: from='client.54576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[67349]: Upgrade: osd.7 is safe to restart 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[67349]: from='client.64522 172.21.15.27:0/2629881563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[71738]: from='client.54576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:18:38.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[71738]: Upgrade: osd.7 is safe to restart 2024-09-17T09:18:38.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:38 smithi027 ceph-mon[71738]: from='client.64522 172.21.15.27:0/2629881563' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: from='client.64528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: pgmap v241: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: Upgrade: Updating osd.7 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:40.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[67349]: Deploying daemon osd.7 on smithi105 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: from='client.64528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: pgmap v241: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: Upgrade: Updating osd.7 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:40.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:39 smithi027 ceph-mon[71738]: Deploying daemon osd.7 on smithi105 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: from='client.64528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: pgmap v241: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: Upgrade: Updating osd.7 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:40.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:39 smithi105 ceph-mon[70467]: Deploying daemon osd.7 on smithi105 2024-09-17T09:18:41.925 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:41 smithi105 systemd[1]: Stopping Ceph osd.7 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:18:41.925 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T09:18:41.732+0000 7fd6e9f4d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-17T09:18:41.925 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T09:18:41.732+0000 7fd6e9f4d700 -1 osd.7 112 *** Got signal Terminated *** 2024-09-17T09:18:41.925 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7[33625]: debug 2024-09-17T09:18:41.732+0000 7fd6e9f4d700 -1 osd.7 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:18:41.925 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:41 smithi105 podman[95443]: 2024-09-17 09:18:41.774746991 +0000 UTC m=+0.150088666 container died fda5accbb1e385a94db14374f0e9241965623e80aea3ac683487532fe01c756d (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, org.label-schema.build-date=20201204, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, org.label-schema.name=CentOS Base Image, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD, ceph=True, maintainer=Dimitri Savineau ) 2024-09-17T09:18:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[67349]: pgmap v242: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[67349]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[67349]: osd.7 failed (root=default,host=smithi105) (connection refused reported by osd.2) 2024-09-17T09:18:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[67349]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[71738]: pgmap v242: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:42.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[71738]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[71738]: osd.7 failed (root=default,host=smithi105) (connection refused reported by osd.2) 2024-09-17T09:18:42.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:41 smithi027 ceph-mon[71738]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:42 smithi105 podman[95443]: 2024-09-17 09:18:42.124960627 +0000 UTC m=+0.500302291 container cleanup fda5accbb1e385a94db14374f0e9241965623e80aea3ac683487532fe01c756d (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7, maintainer=Dimitri Savineau , org.label-schema.build-date=20201204, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=-16.2.0, RELEASE=HEAD, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T09:18:42.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:42 smithi105 bash[95443]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7 2024-09-17T09:18:42.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-mon[70467]: pgmap v242: 161 pgs: 161 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:42.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-mon[70467]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-mon[70467]: osd.7 failed (root=default,host=smithi105) (connection refused reported by osd.2) 2024-09-17T09:18:42.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:41 smithi105 ceph-mon[70467]: osd.7 reported immediately failed by osd.2 2024-09-17T09:18:42.778 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:42 smithi105 podman[95455]: 2024-09-17 09:18:42.392925661 +0000 UTC m=+0.620732614 container remove fda5accbb1e385a94db14374f0e9241965623e80aea3ac683487532fe01c756d (image=quay.io/ceph/ceph@sha256:35b2786dc4cd535dd84f6a1a585503db4b43623ba6c43120b5e5e00951949000, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd.7, CEPH_POINT_RELEASE=-16.2.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=20cb635043d86d971f028e2bf2824446440b5557, RELEASE=HEAD, org.label-schema.name=CentOS Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20201204, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Dimitri Savineau ) 2024-09-17T09:18:43.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:42 smithi105 ceph-mon[70467]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:43.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:42 smithi105 ceph-mon[70467]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2024-09-17T09:18:43.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:42 smithi105 ceph-mon[70467]: osdmap e113: 8 total, 7 up, 8 in 2024-09-17T09:18:43.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:42 smithi105 ceph-mon[70467]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T09:18:43.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:42 smithi105 podman[95571]: 2024-09-17 09:18:42.782648888 +0000 UTC m=+0.016022140 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:43.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:42 smithi105 podman[95571]: 2024-09-17 09:18:42.910937117 +0000 UTC m=+0.144310400 container create bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T09:18:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[71738]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[71738]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2024-09-17T09:18:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[71738]: osdmap e113: 8 total, 7 up, 8 in 2024-09-17T09:18:43.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[71738]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T09:18:43.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[67349]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:18:43.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[67349]: Health check failed: all OSDs are running quincy or later but require_osd_release < quincy (OSD_UPGRADE_FINISHED) 2024-09-17T09:18:43.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[67349]: osdmap e113: 8 total, 7 up, 8 in 2024-09-17T09:18:43.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:42 smithi027 ceph-mon[67349]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T09:18:43.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 podman[95571]: 2024-09-17 09:18:43.155607834 +0000 UTC m=+0.388981076 container init bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:18:43.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 podman[95571]: 2024-09-17 09:18:43.160150245 +0000 UTC m=+0.393523487 container start bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-09-17T09:18:43.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 podman[95571]: 2024-09-17 09:18:43.26532383 +0000 UTC m=+0.498697084 container attach bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:18:43.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 podman[95571]: 2024-09-17 09:18:43.344695519 +0000 UTC m=+0.578068770 container died bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T09:18:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:43 smithi027 ceph-mon[67349]: pgmap v244: 161 pgs: 13 peering, 22 stale+active+clean, 126 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:44.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:43 smithi027 ceph-mon[67349]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:18:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:43 smithi027 ceph-mon[71738]: pgmap v244: 161 pgs: 13 peering, 22 stale+active+clean, 126 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:44.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:43 smithi027 ceph-mon[71738]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:18:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:43 smithi105 ceph-mon[70467]: pgmap v244: 161 pgs: 13 peering, 22 stale+active+clean, 126 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:44.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:43 smithi105 ceph-mon[70467]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:18:44.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 podman[95595]: 2024-09-17 09:18:43.909564518 +0000 UTC m=+0.554637091 container remove bcf0b2c2223d32a0ca3bf43bce9a311b3443b7d1354fc9a165cb79fd5df2c489 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-17T09:18:44.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T09:18:44.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 systemd[1]: Stopped Ceph osd.7 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:18:44.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:43 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.7.service: Consumed 1.148s CPU time. 2024-09-17T09:18:44.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 systemd[1]: Starting Ceph osd.7 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:18:44.639 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.236716894 +0000 UTC m=+0.015788890 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:44.639 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.33562071 +0000 UTC m=+0.114692730 container create 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-17T09:18:44.639 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.58003661 +0000 UTC m=+0.359108602 container init 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=reef-19e2a2b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T09:18:44.639 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.583529234 +0000 UTC m=+0.362601208 container start 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0) 2024-09-17T09:18:44.639 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.638300518 +0000 UTC m=+0.417372514 container attach 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T09:18:44.900 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:18:44.900 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:18:44.900 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:18:44.900 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:18:44.900 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:18:44.901 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:18:45.217 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-mon[70467]: Health check failed: Degraded data redundancy: 53/999 objects degraded (5.305%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate[95709]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 bash[95695]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T09:18:45.218 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:44 smithi105 podman[95695]: 2024-09-17 09:18:44.92555457 +0000 UTC m=+0.704626550 container died 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-17T09:18:45.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:44 smithi027 ceph-mon[67349]: Health check failed: Degraded data redundancy: 53/999 objects degraded (5.305%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:45.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:44 smithi027 ceph-mon[71738]: Health check failed: Degraded data redundancy: 53/999 objects degraded (5.305%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:45.882 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:45 smithi105 podman[95837]: 2024-09-17 09:18:45.55031706 +0000 UTC m=+0.618861246 container remove 1a0f5454fef708c37ee4a83ed68bd3e8c161338de5b04206ce9b5167e6b2d7a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T09:18:45.882 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:45 smithi105 podman[95896]: 2024-09-17 09:18:45.659613545 +0000 UTC m=+0.016305006 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:18:45.882 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:45 smithi105 podman[95896]: 2024-09-17 09:18:45.767896592 +0000 UTC m=+0.124588040 container create 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:18:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:45 smithi027 ceph-mon[67349]: pgmap v246: 161 pgs: 4 active+undersized, 24 peering, 10 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 53/999 objects degraded (5.305%) 2024-09-17T09:18:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:45 smithi027 ceph-mon[71738]: pgmap v246: 161 pgs: 4 active+undersized, 24 peering, 10 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 53/999 objects degraded (5.305%) 2024-09-17T09:18:46.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:45 smithi105 ceph-mon[70467]: pgmap v246: 161 pgs: 4 active+undersized, 24 peering, 10 stale+active+clean, 7 active+undersized+degraded, 116 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 53/999 objects degraded (5.305%) 2024-09-17T09:18:46.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:45 smithi105 podman[95896]: 2024-09-17 09:18:45.995141831 +0000 UTC m=+0.351833285 container init 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-19e2a2b) 2024-09-17T09:18:46.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:45 smithi105 podman[95896]: 2024-09-17 09:18:45.998277412 +0000 UTC m=+0.354968860 container start 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b) 2024-09-17T09:18:46.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:46 smithi105 ceph-osd[95915]: -- 172.21.15.105:0/4170547826 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ec6c77f1e0 con 0x55ec6d4fe000 2024-09-17T09:18:46.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:46 smithi105 bash[95896]: 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 2024-09-17T09:18:46.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:46 smithi105 systemd[1]: Started Ceph osd.7 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:18:46.804 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:46 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:18:46.533+0000 7f0f0c2d0640 -1 Falling back to public interface 2024-09-17T09:18:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[67349]: pgmap v247: 161 pgs: 21 active+undersized, 24 peering, 3 stale+active+clean, 14 active+undersized+degraded, 99 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 73/999 objects degraded (7.307%) 2024-09-17T09:18:47.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:47.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:47 smithi027 ceph-mon[71738]: pgmap v247: 161 pgs: 21 active+undersized, 24 peering, 3 stale+active+clean, 14 active+undersized+degraded, 99 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 73/999 objects degraded (7.307%) 2024-09-17T09:18:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:47 smithi105 ceph-mon[70467]: pgmap v247: 161 pgs: 21 active+undersized, 24 peering, 3 stale+active+clean, 14 active+undersized+degraded, 99 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 73/999 objects degraded (7.307%) 2024-09-17T09:18:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:48 smithi105 ceph-mon[70467]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:18:48.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:48 smithi027 ceph-mon[71738]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:18:48.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:48 smithi027 ceph-mon[67349]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:18:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-mon[70467]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-mon[70467]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-mon[70467]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:49.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:18:49.249+0000 7f0f0c2d0640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T09:18:49.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:18:49.279+0000 7f0f0c2d0640 -1 osd.7 112 log_to_monitors true 2024-09-17T09:18:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[67349]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[67349]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:49.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[67349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:49.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[71738]: pgmap v248: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 273 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:49.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[71738]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:49.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:49 smithi027 ceph-mon[71738]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:18:50.028 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:18:49 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:18:49.869+0000 7f0f00072640 -1 osd.7 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:18:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:50 smithi105 ceph-mon[70467]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:18:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:50 smithi105 ceph-mon[70467]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T09:18:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:50 smithi105 ceph-mon[70467]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:50 smithi105 ceph-mon[70467]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:50 smithi105 ceph-mon[70467]: from='osd.7 ' entity='osd.7' 2024-09-17T09:18:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[67349]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:18:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[67349]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T09:18:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[67349]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[67349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[67349]: from='osd.7 ' entity='osd.7' 2024-09-17T09:18:50.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[71738]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:18:50.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[71738]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T09:18:50.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[71738]: from='osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[71738]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-17T09:18:50.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:50 smithi027 ceph-mon[71738]: from='osd.7 ' entity='osd.7' 2024-09-17T09:18:51.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: pgmap v250: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:51.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:51.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276] boot 2024-09-17T09:18:51.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T09:18:51.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:18:51.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:51 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: pgmap v250: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276] boot 2024-09-17T09:18:51.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: pgmap v250: 161 pgs: 45 active+undersized, 24 active+undersized+degraded, 92 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 127/999 objects degraded (12.713%) 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: osd.7 [v2:172.21.15.105:6824/988799276,v1:172.21.15.105:6825/988799276] boot 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:18:51.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:51.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:51 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:53.205 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:52 smithi105 ceph-mon[70467]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T09:18:53.205 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:52 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 127/999 objects degraded (12.713%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:52 smithi027 ceph-mon[67349]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T09:18:53.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:52 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 127/999 objects degraded (12.713%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:52 smithi027 ceph-mon[71738]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T09:18:53.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:52 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 127/999 objects degraded (12.713%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: pgmap v253: 161 pgs: 7 peering, 34 active+undersized, 16 active+undersized+degraded, 104 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 93/999 objects degraded (9.309%) 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:54.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: pgmap v253: 161 pgs: 7 peering, 34 active+undersized, 16 active+undersized+degraded, 104 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 93/999 objects degraded (9.309%) 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:54.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:53 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: pgmap v253: 161 pgs: 7 peering, 34 active+undersized, 16 active+undersized+degraded, 104 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 93/999 objects degraded (9.309%) 2024-09-17T09:18:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:54.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:54.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:54.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:53 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:55.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: pgmap v254: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: pgmap v255: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.159 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all osd 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T09:18:55.160 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:55.161 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: pgmap v254: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: pgmap v255: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all osd 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T09:18:55.162 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T09:18:55.163 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: Upgrade: Setting require_osd_release to 18 reef 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: Upgrade: Setting require_osd_release to 18 reef 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-17T09:18:55.164 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:54 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: pgmap v254: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: pgmap v255: 161 pgs: 7 peering, 27 active+undersized, 13 active+undersized+degraded, 114 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 75/999 objects degraded (7.508%) 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all osd 2024-09-17T09:18:55.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T09:18:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: Upgrade: Setting require_osd_release to 18 reef 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "reef"}]: dispatch 2024-09-17T09:18:55.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:54 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2024-09-17T09:18:56.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:18:56.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:18:56.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:18:56.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.268 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.269 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.270 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:55 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running quincy or later but require_osd_release < quincy) 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "reef"}]': finished 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:18:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:18:56.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:18:56.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:18:56.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:55 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:18:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mds 2024-09-17T09:18:57.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nfs 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: pgmap v257: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: pgmap v258: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: pgmap v259: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[67349]: pgmap v260: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mds 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nfs 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: pgmap v257: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: pgmap v258: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.260 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: pgmap v259: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:56 smithi027 ceph-mon[71738]: pgmap v260: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mds 2024-09-17T09:18:57.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nfs 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: pgmap v257: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: pgmap v258: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: pgmap v259: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:57.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:56 smithi105 ceph-mon[70467]: pgmap v260: 161 pgs: 7 peering, 5 active+undersized, 2 active+undersized+degraded, 147 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 3/999 objects degraded (0.300%) 2024-09-17T09:18:58.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:57 smithi027 ceph-mon[67349]: Health check update: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:58.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:57 smithi027 ceph-mon[71738]: Health check update: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:58.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:57 smithi105 ceph-mon[70467]: Health check update: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:18:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:58 smithi027 ceph-mon[67349]: pgmap v261: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:59.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:18:58 smithi027 ceph-mon[67349]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded) 2024-09-17T09:18:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:58 smithi027 ceph-mon[71738]: pgmap v261: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:59.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:18:58 smithi027 ceph-mon[71738]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded) 2024-09-17T09:18:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:58 smithi105 ceph-mon[70467]: pgmap v261: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:18:59.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:18:58 smithi105 ceph-mon[70467]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/999 objects degraded (0.300%), 2 pgs degraded) 2024-09-17T09:19:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:00 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:00.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:00 smithi027 ceph-mon[67349]: pgmap v262: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:00 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:00.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:00 smithi027 ceph-mon[71738]: pgmap v262: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:00 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:00 smithi105 ceph-mon[70467]: pgmap v262: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:02.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:02 smithi027 ceph-mon[67349]: pgmap v263: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:02.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:02 smithi027 ceph-mon[71738]: pgmap v263: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:02 smithi105 ceph-mon[70467]: pgmap v263: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:04 smithi027 ceph-mon[67349]: pgmap v264: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:04.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:04 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:04.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:04 smithi027 ceph-mon[71738]: pgmap v264: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:04.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:04 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:04 smithi105 ceph-mon[70467]: pgmap v264: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:04 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:06.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:06 smithi027 ceph-mon[67349]: pgmap v265: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:06.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:06 smithi027 ceph-mon[71738]: pgmap v265: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:06 smithi105 ceph-mon[70467]: pgmap v265: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:08.744 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:08 smithi027 ceph-mon[67349]: pgmap v266: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:08.744 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:08 smithi027 ceph-mon[71738]: pgmap v266: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:08.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:08 smithi105 ceph-mon[70467]: pgmap v266: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:09.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:19:09.446 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:09 smithi027 ceph-mon[67349]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:09.446 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:09 smithi027 ceph-mon[71738]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:09.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:09 smithi105 ceph-mon[70467]: from='client.64537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:10.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:10 smithi027 ceph-mon[71738]: pgmap v267: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:10.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:10 smithi027 ceph-mon[67349]: pgmap v267: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:10.679 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:10 smithi105 ceph-mon[70467]: pgmap v267: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:11.178 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 2m ago 30m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (8m) 20s ago 30m 90.0M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 20s ago 29m - - 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (9m) 20s ago 34m 438M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (19m) 2m ago 35m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 2m ago 35m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (7m) 20s ago 34m 57.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 2m ago 34m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 30m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (9m) 20s ago 30m 17.1M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (4m) 2m ago 33m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:19:11.179 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (4m) 2m ago 33m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (3m) 2m ago 33m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (2m) 2m ago 33m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (2m) 20s ago 32m 53.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (89s) 20s ago 32m 52.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (57s) 20s ago 32m 52.8M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (25s) 20s ago 31m 12.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (8m) 20s ago 31m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (29m) 2m ago 29m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:19:11.180 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (29m) 20s ago 29m 45.7M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:19:11.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-09-17T09:19:12.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:12 smithi027 ceph-mon[67349]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:12.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:12 smithi027 ceph-mon[67349]: pgmap v268: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:12.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:12 smithi027 ceph-mon[71738]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:12.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:12 smithi027 ceph-mon[71738]: pgmap v268: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:12.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:12 smithi105 ceph-mon[70467]: from='client.64543 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:12.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:12 smithi105 ceph-mon[70467]: pgmap v268: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:13.927 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:19:14.618 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:14 smithi027 ceph-mon[67349]: pgmap v269: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:14.618 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:14 smithi027 ceph-mon[67349]: from='client.54609 172.21.15.27:0/3148323919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:14.618 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:14 smithi027 ceph-mon[71738]: pgmap v269: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:14.619 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:14 smithi027 ceph-mon[71738]: from='client.54609 172.21.15.27:0/3148323919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:14.641 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-09-17T09:19:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:14 smithi105 ceph-mon[70467]: pgmap v269: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:14.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:14 smithi105 ceph-mon[70467]: from='client.54609 172.21.15.27:0/3148323919' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:16.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:16 smithi027 ceph-mon[67349]: pgmap v270: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:16.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:16 smithi027 ceph-mon[71738]: pgmap v270: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:16.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:16 smithi105 ceph-mon[70467]: pgmap v270: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:16.794 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)" 2024-09-17T09:19:17.442 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:17 smithi027 ceph-mon[71738]: from='client.54615 172.21.15.27:0/2724726416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:17.443 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:17 smithi027 ceph-mon[67349]: from='client.54615 172.21.15.27:0/2724726416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:17.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:19:17.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:17 smithi105 ceph-mon[70467]: from='client.54615 172.21.15.27:0/2724726416' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:18.652 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:18 smithi027 ceph-mon[67349]: pgmap v271: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s 2024-09-17T09:19:18.652 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:18 smithi027 ceph-mon[71738]: pgmap v271: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s 2024-09-17T09:19:18.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:18 smithi105 ceph-mon[70467]: pgmap v271: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s 2024-09-17T09:19:19.366 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:19:19.366 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:19:19.366 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:19:19.366 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:19:19.367 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:19:19.367 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:19:19.367 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:19:19.367 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:19:19.367 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:19:19.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:19:20.604 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:20 smithi027 ceph-mon[67349]: pgmap v272: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:19:20.605 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:20 smithi027 ceph-mon[67349]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:20.605 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:20 smithi027 ceph-mon[71738]: pgmap v272: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:19:20.605 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:20 smithi027 ceph-mon[71738]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:20 smithi105 ceph-mon[70467]: pgmap v272: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 0 B/s wr, 9 op/s 2024-09-17T09:19:20.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:20 smithi105 ceph-mon[70467]: from='client.64561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:22.106 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:19:22.106 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:19:22.107 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in error state 2024-09-17T09:19:22.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:22 smithi027 ceph-mon[67349]: pgmap v273: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 43 op/s 2024-09-17T09:19:22.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:22 smithi027 ceph-mon[67349]: from='client.64567 172.21.15.27:0/1011940851' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:19:22.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:22 smithi027 ceph-mon[71738]: pgmap v273: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 43 op/s 2024-09-17T09:19:22.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:22 smithi027 ceph-mon[71738]: from='client.64567 172.21.15.27:0/1011940851' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:19:22.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:22 smithi105 ceph-mon[70467]: pgmap v273: 161 pgs: 161 active+clean; 604 KiB data, 292 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 43 op/s 2024-09-17T09:19:22.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:22 smithi105 ceph-mon[70467]: from='client.64567 172.21.15.27:0/1011940851' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:19:22.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-09-17T09:19:24.380 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:24 smithi027 ceph-mon[67349]: pgmap v274: 161 pgs: 161 active+clean; 604 KiB data, 296 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 95 op/s 2024-09-17T09:19:24.380 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:24 smithi027 ceph-mon[71738]: pgmap v274: 161 pgs: 161 active+clean; 604 KiB data, 296 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 95 op/s 2024-09-17T09:19:24.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:24 smithi105 ceph-mon[70467]: pgmap v274: 161 pgs: 161 active+clean; 604 KiB data, 296 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 0 B/s wr, 95 op/s 2024-09-17T09:19:25.455 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:25 smithi027 ceph-mon[67349]: from='client.64573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:25.456 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:25 smithi027 ceph-mon[71738]: from='client.64573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:25.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:25 smithi105 ceph-mon[70467]: from='client.64573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:26.096 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:26.688 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: pgmap v275: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 117 op/s 2024-09-17T09:19:26.688 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:26.688 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:26.688 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:26.688 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: pgmap v275: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 117 op/s 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:26.689 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:26.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-17T09:19:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: pgmap v275: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 0 B/s wr, 117 op/s 2024-09-17T09:19:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:26.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:26.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:26.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:27.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[67349]: pgmap v276: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 0 B/s wr, 131 op/s 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[67349]: pgmap v277: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[71738]: pgmap v276: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 0 B/s wr, 131 op/s 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[71738]: pgmap v277: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:19:27.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:27 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:27 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:27 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:27.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:27 smithi105 ceph-mon[70467]: pgmap v276: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 0 B/s wr, 131 op/s 2024-09-17T09:19:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:27 smithi105 ceph-mon[70467]: pgmap v277: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 0 B/s wr, 160 op/s 2024-09-17T09:19:27.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:27 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:19:29.610 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:19:29.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: pgmap v278: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 149 op/s 2024-09-17T09:19:29.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.610 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.611 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: pgmap v278: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 149 op/s 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:29.612 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:29.613 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:29 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: pgmap v278: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 0 B/s wr, 149 op/s 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:30.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:29 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (8m) 2m ago 30m 24.6M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (8m) 39s ago 30m 90.0M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 39s ago 30m - - 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (9m) 39s ago 34m 438M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (20m) 2m ago 35m 483M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (6m) 2m ago 36m 58.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (7m) 39s ago 34m 57.2M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 2m ago 34m 47.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (9m) 2m ago 31m 16.4M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:19:30.282 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (9m) 39s ago 31m 17.1M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (5m) 2m ago 34m 52.6M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (4m) 2m ago 33m 44.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (3m) 2m ago 33m 41.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (2m) 2m ago 33m 11.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (2m) 39s ago 33m 53.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (108s) 39s ago 32m 52.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (76s) 39s ago 32m 52.8M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (44s) 39s ago 32m 12.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (9m) 39s ago 31m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (30m) 2m ago 30m 43.6M - 16.2.0 24ecd6d5f14c 849da5aebed5 2024-09-17T09:19:30.283 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (30m) 39s ago 30m 45.7M - 16.2.0 24ecd6d5f14c 33b7285d7713 2024-09-17T09:19:30.690 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:19:30.690 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:19:30.690 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:19:30.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:19:30.690 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 8 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2 2024-09-17T09:19:30.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:19:30.692 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:19:30.692 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 16.2.0 (0c2054e95bcd9b30fdd908a79ac1d8bbc3394442) pacific (stable)": 2, 2024-09-17T09:19:30.692 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 13 2024-09-17T09:19:30.692 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:19:30.692 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:19:30.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:19:30.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:19:30.900 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all osd 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mds 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[67349]: from='client.64585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:19:30.901 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:19:30.902 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all osd 2024-09-17T09:19:30.902 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mds 2024-09-17T09:19:30.902 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:30.902 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:30 smithi027 ceph-mon[71738]: from='client.64585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:19:31.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all osd 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mds 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: from='client.64579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:30 smithi105 ceph-mon[70467]: from='client.64585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "0/2 daemons upgraded", 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading rgw daemons", 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:19:31.033 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:19:31.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: pgmap v279: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 102 op/s 2024-09-17T09:19:31.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='client.64591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: Upgrade: Updating rgw.foo.smithi027.yrcgts (1/2) 2024-09-17T09:19:31.888 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='client.54657 172.21.15.27:0/2371302748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[67349]: from='client.64603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: pgmap v279: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 102 op/s 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='client.64591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: Upgrade: Updating rgw.foo.smithi027.yrcgts (1/2) 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:31.889 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:31.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:31.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:19:31.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='client.54657 172.21.15.27:0/2371302748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:31.890 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:31 smithi027 ceph-mon[71738]: from='client.64603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: pgmap v279: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 0 B/s wr, 102 op/s 2024-09-17T09:19:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='client.64591 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: Upgrade: Updating rgw.foo.smithi027.yrcgts (1/2) 2024-09-17T09:19:32.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:32.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi027.yrcgts", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:32.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:32.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: Deploying daemon rgw.foo.smithi027.yrcgts on smithi027 2024-09-17T09:19:32.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='client.54657 172.21.15.27:0/2371302748' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:32.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:31 smithi105 ceph-mon[70467]: from='client.64603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:19:33.749 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:33 smithi027 ceph-mon[67349]: pgmap v280: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 32 op/s 2024-09-17T09:19:33.750 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:33 smithi027 ceph-mon[71738]: pgmap v280: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 32 op/s 2024-09-17T09:19:34.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:33 smithi105 ceph-mon[70467]: pgmap v280: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 32 op/s 2024-09-17T09:19:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[67349]: pgmap v281: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-09-17T09:19:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:35.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[71738]: pgmap v281: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-09-17T09:19:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:35 smithi105 ceph-mon[70467]: pgmap v281: 161 pgs: 161 active+clean; 604 KiB data, 300 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-09-17T09:19:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:36 smithi105 ceph-mon[70467]: Upgrade: Updating rgw.foo.smithi105.afiseg (2/2) 2024-09-17T09:19:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:36 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:36 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:37.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:36 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:37.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:36 smithi105 ceph-mon[70467]: Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:19:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[67349]: Upgrade: Updating rgw.foo.smithi105.afiseg (2/2) 2024-09-17T09:19:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:37.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[67349]: Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:19:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[71738]: Upgrade: Updating rgw.foo.smithi105.afiseg (2/2) 2024-09-17T09:19:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi105.afiseg", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:19:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:37.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:36 smithi027 ceph-mon[71738]: Deploying daemon rgw.foo.smithi105.afiseg on smithi105 2024-09-17T09:19:38.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:37 smithi027 ceph-mon[67349]: pgmap v282: 161 pgs: 161 active+clean; 604 KiB data, 304 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 204 B/s wr, 48 op/s 2024-09-17T09:19:38.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:37 smithi027 ceph-mon[71738]: pgmap v282: 161 pgs: 161 active+clean; 604 KiB data, 304 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 204 B/s wr, 48 op/s 2024-09-17T09:19:38.103 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:37 smithi105 ceph-mon[70467]: pgmap v282: 161 pgs: 161 active+clean; 604 KiB data, 304 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 204 B/s wr, 48 op/s 2024-09-17T09:19:40.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:39 smithi105 ceph-mon[70467]: pgmap v283: 161 pgs: 161 active+clean; 604 KiB data, 308 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 170 B/s wr, 93 op/s 2024-09-17T09:19:40.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.104 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:39 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:40.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[71738]: pgmap v283: 161 pgs: 161 active+clean; 604 KiB data, 308 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 170 B/s wr, 93 op/s 2024-09-17T09:19:40.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.153 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:40.154 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[67349]: pgmap v283: 161 pgs: 161 active+clean; 604 KiB data, 308 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 170 B/s wr, 93 op/s 2024-09-17T09:19:40.154 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.154 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:40.154 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:39 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:42.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:41 smithi105 ceph-mon[70467]: pgmap v284: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 103 op/s 2024-09-17T09:19:42.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:41 smithi027 ceph-mon[67349]: pgmap v284: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 103 op/s 2024-09-17T09:19:42.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:41 smithi027 ceph-mon[71738]: pgmap v284: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 170 B/s wr, 103 op/s 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: pgmap v285: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 255 B/s wr, 129 op/s 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.505 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: pgmap v285: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 255 B/s wr, 129 op/s 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.506 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:43 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: pgmap v285: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 255 B/s wr, 129 op/s 2024-09-17T09:19:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:43.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:43 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:44 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:44 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.244 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:44 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.245 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:44 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:44 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:45.245 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:44 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:46.111 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:45 smithi105 ceph-mon[70467]: pgmap v286: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 255 B/s wr, 159 op/s 2024-09-17T09:19:46.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:45 smithi027 ceph-mon[67349]: pgmap v286: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 255 B/s wr, 159 op/s 2024-09-17T09:19:46.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:45 smithi027 ceph-mon[71738]: pgmap v286: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 255 B/s wr, 159 op/s 2024-09-17T09:19:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]: dispatch 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]': finished 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]: dispatch 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]': finished 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:46 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]: dispatch 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]': finished 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]: dispatch 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]': finished 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi027.yrcgts"}]': finished 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi105.afiseg"}]': finished 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:47.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:47.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:46 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: pgmap v287: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 255 B/s wr, 186 op/s 2024-09-17T09:19:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: pgmap v288: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s rd, 95 B/s wr, 164 op/s 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: pgmap v289: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 117 B/s wr, 127 op/s 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rgw 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nfs 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:19:48.260 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: pgmap v287: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 255 B/s wr, 186 op/s 2024-09-17T09:19:48.261 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: pgmap v288: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s rd, 95 B/s wr, 164 op/s 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: pgmap v289: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 117 B/s wr, 127 op/s 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rgw 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nfs 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.262 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.263 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:19:48.264 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:19:48.265 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:19:48.266 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:19:48.267 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.268 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.269 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: pgmap v290: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 147 B/s wr, 143 op/s 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: pgmap v291: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 139 op/s 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: Checking dashboard <-> RGW credentials 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.270 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:19:48.271 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: pgmap v290: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 147 B/s wr, 143 op/s 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: pgmap v291: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 139 op/s 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: Checking dashboard <-> RGW credentials 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.272 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.273 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.273 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.273 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:47 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: pgmap v287: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 117 KiB/s rd, 255 B/s wr, 186 op/s 2024-09-17T09:19:48.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: pgmap v288: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s rd, 95 B/s wr, 164 op/s 2024-09-17T09:19:48.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: pgmap v289: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 117 B/s wr, 127 op/s 2024-09-17T09:19:48.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rgw 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nfs 2024-09-17T09:19:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:19:48.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:19:48.283 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:19:48.284 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:19:48.285 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:19:48.286 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.287 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: pgmap v290: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 147 B/s wr, 143 op/s 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: pgmap v291: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 0 B/s wr, 139 op/s 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: Checking dashboard <-> RGW credentials 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:48.288 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:19:48.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:19:48.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:19:48.289 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:47 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[67349]: pgmap v292: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 99 op/s 2024-09-17T09:19:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[67349]: pgmap v293: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:49.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[71738]: pgmap v292: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 99 op/s 2024-09-17T09:19:49.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:49 smithi027 ceph-mon[71738]: pgmap v293: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:49 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:19:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:49 smithi105 ceph-mon[70467]: pgmap v292: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 0 B/s wr, 99 op/s 2024-09-17T09:19:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:49 smithi105 ceph-mon[70467]: pgmap v293: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:19:50.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:50 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:50.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:50 smithi027 ceph-mon[67349]: pgmap v294: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 389 B/s wr, 1 op/s 2024-09-17T09:19:50.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:50 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:50.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:50 smithi027 ceph-mon[71738]: pgmap v294: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 389 B/s wr, 1 op/s 2024-09-17T09:19:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:50 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:19:50.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:50 smithi105 ceph-mon[70467]: pgmap v294: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 389 B/s wr, 1 op/s 2024-09-17T09:19:52.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:52 smithi027 ceph-mon[67349]: pgmap v295: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 442 B/s wr, 5 op/s 2024-09-17T09:19:52.822 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:52 smithi027 ceph-mon[71738]: pgmap v295: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 442 B/s wr, 5 op/s 2024-09-17T09:19:52.822 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:52 smithi105 ceph-mon[70467]: pgmap v295: 161 pgs: 161 active+clean; 604 KiB data, 312 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 442 B/s wr, 5 op/s 2024-09-17T09:19:54.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:54 smithi027 ceph-mon[67349]: pgmap v296: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 319 B/s wr, 4 op/s 2024-09-17T09:19:54.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:54 smithi027 ceph-mon[71738]: pgmap v296: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 319 B/s wr, 4 op/s 2024-09-17T09:19:54.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:54 smithi105 ceph-mon[70467]: pgmap v296: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 319 B/s wr, 4 op/s 2024-09-17T09:19:56.565 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:56 smithi027 ceph-mon[67349]: pgmap v297: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 243 B/s wr, 3 op/s 2024-09-17T09:19:56.565 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:56 smithi027 ceph-mon[71738]: pgmap v297: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 243 B/s wr, 3 op/s 2024-09-17T09:19:56.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:56 smithi105 ceph-mon[70467]: pgmap v297: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 243 B/s wr, 3 op/s 2024-09-17T09:19:58.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:19:58 smithi027 ceph-mon[67349]: pgmap v298: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 204 B/s wr, 2 op/s 2024-09-17T09:19:58.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:19:58 smithi027 ceph-mon[71738]: pgmap v298: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 204 B/s wr, 2 op/s 2024-09-17T09:19:58.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:19:58 smithi105 ceph-mon[70467]: pgmap v298: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 204 B/s wr, 2 op/s 2024-09-17T09:20:00.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:00 smithi027 ceph-mon[67349]: pgmap v299: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s 2024-09-17T09:20:00.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:00 smithi027 ceph-mon[67349]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:20:00.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:00 smithi027 ceph-mon[71738]: pgmap v299: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s 2024-09-17T09:20:00.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:00 smithi027 ceph-mon[71738]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:20:00.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:00 smithi105 ceph-mon[70467]: pgmap v299: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s 2024-09-17T09:20:00.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:00 smithi105 ceph-mon[70467]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:20:02.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:20:02.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:02 smithi027 ceph-mon[71738]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:02.709 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:02 smithi027 ceph-mon[71738]: pgmap v300: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T09:20:02.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:02 smithi027 ceph-mon[67349]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:02.709 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:02 smithi027 ceph-mon[67349]: pgmap v300: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T09:20:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:02 smithi105 ceph-mon[70467]: from='client.64711 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:02.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:02 smithi105 ceph-mon[70467]: pgmap v300: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 85 B/s wr, 1 op/s 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 21s ago 31m 24.2M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (9m) 20s ago 30m 91.0M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 20s ago 30m - - 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (10m) 20s ago 35m 439M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (20m) 21s ago 36m 489M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (7m) 21s ago 36m 76.4M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (8m) 20s ago 35m 60.7M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:20:04.038 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (6m) 21s ago 35m 54.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (10m) 21s ago 31m 16.6M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (10m) 20s ago 31m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (5m) 21s ago 34m 61.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (5m) 21s ago 34m 54.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (4m) 21s ago 34m 48.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (3m) 21s ago 33m 57.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (2m) 20s ago 33m 59.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (2m) 20s ago 33m 56.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (110s) 20s ago 33m 56.1M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (78s) 20s ago 32m 55.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:20:04.039 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (9m) 20s ago 32m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:20:04.040 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (30s) 21s ago 30m 76.7M - 18.2.4-855-g19e2a2be 980a0bfcd684 e5c240bc34bc 2024-09-17T09:20:04.040 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (25s) 20s ago 30m 74.6M - 18.2.4-855-g19e2a2be 980a0bfcd684 f99988b09038 2024-09-17T09:20:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:04 smithi027 ceph-mon[71738]: pgmap v301: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:04.686 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:04 smithi027 ceph-mon[71738]: from='client.54768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:04.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:04 smithi027 ceph-mon[67349]: pgmap v301: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:04.687 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:04 smithi027 ceph-mon[67349]: from='client.54768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:04.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-09-17T09:20:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:04 smithi105 ceph-mon[70467]: pgmap v301: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:04.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:04 smithi105 ceph-mon[70467]: from='client.54768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:06.680 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:06 smithi027 ceph-mon[67349]: pgmap v302: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:06.681 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:06 smithi027 ceph-mon[71738]: pgmap v302: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:06.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:06 smithi105 ceph-mon[70467]: pgmap v302: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:06.929 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:20:07.534 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:07 smithi027 ceph-mon[67349]: from='client.64723 172.21.15.27:0/3967547904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:07.534 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:07 smithi027 ceph-mon[71738]: from='client.64723 172.21.15.27:0/3967547904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:07.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-09-17T09:20:07.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:07 smithi105 ceph-mon[70467]: from='client.64723 172.21.15.27:0/3967547904' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:08.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:08 smithi105 ceph-mon[70467]: pgmap v303: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:08.816 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:08 smithi027 ceph-mon[71738]: pgmap v303: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:08.817 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:08 smithi027 ceph-mon[67349]: pgmap v303: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:09.601 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)" 2024-09-17T09:20:10.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:20:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:10 smithi027 ceph-mon[67349]: pgmap v304: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:10.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:10 smithi027 ceph-mon[67349]: from='client.64729 172.21.15.27:0/4129338069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:10 smithi027 ceph-mon[71738]: pgmap v304: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:10.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:10 smithi027 ceph-mon[71738]: from='client.64729 172.21.15.27:0/4129338069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:10 smithi105 ceph-mon[70467]: pgmap v304: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:10.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:10 smithi105 ceph-mon[70467]: from='client.64729 172.21.15.27:0/4129338069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:20:12.392 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:20:12.713 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:12 smithi027 ceph-mon[67349]: pgmap v305: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:12.714 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:12 smithi027 ceph-mon[71738]: pgmap v305: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:12.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:12 smithi105 ceph-mon[70467]: pgmap v305: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:13.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:20:13.758 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:13 smithi027 ceph-mon[67349]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:13.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:13 smithi027 ceph-mon[71738]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:14.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:13 smithi105 ceph-mon[70467]: from='client.64735 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:14.693 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:14 smithi027 ceph-mon[67349]: pgmap v306: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:14.693 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:14 smithi027 ceph-mon[71738]: pgmap v306: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:15.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:14 smithi105 ceph-mon[70467]: pgmap v306: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:15.242 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:20:15.242 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:20:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in error state 2024-09-17T09:20:15.979 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:15 smithi027 ceph-mon[71738]: from='client.54792 172.21.15.27:0/4012644195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:15.980 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:15 smithi027 ceph-mon[67349]: from='client.54792 172.21.15.27:0/4012644195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:16.003 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T09:20:16.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:15 smithi105 ceph-mon[70467]: from='client.54792 172.21.15.27:0/4012644195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:17.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:16 smithi027 ceph-mon[67349]: pgmap v307: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:17.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:16 smithi027 ceph-mon[71738]: pgmap v307: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:17.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:16 smithi105 ceph-mon[70467]: pgmap v307: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:18.239 INFO:teuthology.orchestra.run.smithi027.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:18.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: pgmap v308: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:18.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:18.928 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: pgmap v308: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:18.929 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:18 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:18.952 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:20:18.963 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-09-17T09:20:18.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T09:20:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: pgmap v308: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:19.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:19.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:18 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: from='client.64747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: pgmap v309: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: pgmap v310: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:19.994 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[71738]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: from='client.64747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: pgmap v309: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: pgmap v310: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:19.995 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:19 smithi027 ceph-mon[67349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:20.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: from='client.64747 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:20.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:20.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:20.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: pgmap v309: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:20.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: pgmap v310: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:20.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:19 smithi105 ceph-mon[70467]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 2024-09-17T09:20:21.398 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:20:21.671 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:21 smithi027 ceph-mon[71738]: pgmap v311: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:21.672 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:21 smithi027 ceph-mon[67349]: pgmap v311: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:21.834 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:21 smithi105 ceph-mon[70467]: pgmap v311: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:22.085 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (9m) 39s ago 31m 24.2M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (9m) 38s ago 31m 91.0M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 error 38s ago 30m - - 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (10m) 38s ago 35m 439M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (20m) 39s ago 36m 489M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (7m) 39s ago 37m 76.4M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (8m) 38s ago 35m 60.7M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 39s ago 35m 54.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (10m) 39s ago 32m 16.6M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (10m) 38s ago 32m 16.7M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (6m) 39s ago 35m 61.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:20:22.086 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (5m) 39s ago 34m 54.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (4m) 39s ago 34m 48.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (3m) 39s ago 34m 57.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (3m) 38s ago 33m 59.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (2m) 38s ago 33m 56.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (2m) 38s ago 33m 56.1M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (96s) 38s ago 33m 55.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (10m) 38s ago 32m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (48s) 39s ago 31m 76.7M - 18.2.4-855-g19e2a2be 980a0bfcd684 e5c240bc34bc 2024-09-17T09:20:22.087 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (43s) 38s ago 31m 74.6M - 18.2.4-855-g19e2a2be 980a0bfcd684 f99988b09038 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:20:22.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 8 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 15 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:20:22.489 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:20:22.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='client.64753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:22.758 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mgr 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mon 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.759 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all crash 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all osd 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all mds 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rgw 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:20:22.760 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[71738]: from='client.64771 172.21.15.27:0/2806151822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='client.64753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:22.761 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mgr 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mon 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.762 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all crash 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all osd 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all mds 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rgw 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.763 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:22.764 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:22 smithi027 ceph-mon[67349]: from='client.64771 172.21.15.27:0/2806151822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:22.827 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:20:22.827 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": true, 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [ 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "mon", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "osd" 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": "15/21 daemons upgraded", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "Currently upgrading iscsi daemons", 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:20:22.828 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='client.64753 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Target is version 18.2.4-855-g19e2a2be (reef) 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, digests ['quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9'] 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mgr 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mon 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all crash 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all osd 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all mds 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rgw 2024-09-17T09:20:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='client.54810 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:22 smithi105 ceph-mon[70467]: from='client.64771 172.21.15.27:0/2806151822' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:23.247 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-09-17T09:20:23.247 INFO:teuthology.orchestra.run.smithi027.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-09-17T09:20:23.247 INFO:teuthology.orchestra.run.smithi027.stdout: daemon iscsi.foo.smithi105.mzuaqj on smithi105 is in error state 2024-09-17T09:20:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='client.64765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: pgmap v312: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[71738]: from='client.54834 172.21.15.27:0/2903089554' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='client.64765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: pgmap v312: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:20:24.009 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:24.010 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:23 smithi027 ceph-mon[67349]: from='client.54834 172.21.15.27:0/2903089554' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='client.64765 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: pgmap v312: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='client.64777 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:24.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi105.mzuaqj", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-09-17T09:20:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:24.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:23 smithi105 ceph-mon[70467]: from='client.54834 172.21.15.27:0/2903089554' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:20:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:24 smithi027 ceph-mon[67349]: Upgrade: Updating iscsi.foo.smithi105.mzuaqj 2024-09-17T09:20:25.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:24 smithi027 ceph-mon[67349]: Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:20:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:24 smithi027 ceph-mon[71738]: Upgrade: Updating iscsi.foo.smithi105.mzuaqj 2024-09-17T09:20:25.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:24 smithi027 ceph-mon[71738]: Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:20:25.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:24 smithi105 ceph-mon[70467]: Upgrade: Updating iscsi.foo.smithi105.mzuaqj 2024-09-17T09:20:25.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:24 smithi105 ceph-mon[70467]: Deploying daemon iscsi.foo.smithi105.mzuaqj on smithi105 2024-09-17T09:20:25.909 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:25 smithi105 ceph-mon[70467]: pgmap v313: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:26.008 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:25 smithi027 ceph-mon[71738]: pgmap v313: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:26.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:25 smithi027 ceph-mon[67349]: pgmap v313: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/3201090739' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/1683541126' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:27.015 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/3201090739' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/1683541126' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:27.016 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:26 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:27.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/3201090739' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-09-17T09:20:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/1683541126' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:27.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:26 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]: dispatch 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[67349]: pgmap v314: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]': finished 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[67349]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/2139208934' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:28.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[71738]: pgmap v314: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]': finished 2024-09-17T09:20:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[71738]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T09:20:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/2139208934' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:27 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:27 smithi105 ceph-mon[70467]: pgmap v314: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:20:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:27 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/210613837"}]': finished 2024-09-17T09:20:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:27 smithi105 ceph-mon[70467]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T09:20:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:27 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/2139208934' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:28.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:27 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]: dispatch 2024-09-17T09:20:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]': finished 2024-09-17T09:20:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[67349]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T09:20:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/2200164422' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:29.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:29.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]': finished 2024-09-17T09:20:29.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[71738]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T09:20:29.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/2200164422' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:29.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:28 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:28 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3280907434"}]': finished 2024-09-17T09:20:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:28 smithi105 ceph-mon[70467]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T09:20:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:28 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/2200164422' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:29.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:28 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]: dispatch 2024-09-17T09:20:30.077 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:29 smithi105 ceph-mon[70467]: pgmap v317: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T09:20:30.077 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:29 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]': finished 2024-09-17T09:20:30.077 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:29 smithi105 ceph-mon[70467]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T09:20:30.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:29 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/992345899' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:30.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:29 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:30.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[67349]: pgmap v317: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T09:20:30.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]': finished 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[67349]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/992345899' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[71738]: pgmap v317: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3573774270"}]': finished 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[71738]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/992345899' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:30.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:29 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]: dispatch 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]': finished 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/80095148' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:31 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]': finished 2024-09-17T09:20:31.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/80095148' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/2718211569"}]': finished 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/80095148' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]: dispatch 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:31.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:31 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:32 smithi105 ceph-mon[70467]: pgmap v320: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:32 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]': finished 2024-09-17T09:20:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:32 smithi105 ceph-mon[70467]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T09:20:32.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:32 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/2791642999' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:32.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:32 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:32.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[67349]: pgmap v320: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:32.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]': finished 2024-09-17T09:20:32.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[67349]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T09:20:32.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/2791642999' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:32.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:32.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[71738]: pgmap v320: 161 pgs: 161 active+clean; 604 KiB data, 316 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:20:32.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3522830744"}]': finished 2024-09-17T09:20:32.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[71738]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T09:20:32.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/2791642999' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:32.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:32 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]: dispatch 2024-09-17T09:20:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:33 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]': finished 2024-09-17T09:20:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:33 smithi105 ceph-mon[70467]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T09:20:33.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:33 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/1840375144' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:33.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:33 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:33.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]': finished 2024-09-17T09:20:33.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[67349]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T09:20:33.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/1840375144' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:33.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:33.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1688053069"}]': finished 2024-09-17T09:20:33.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[71738]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T09:20:33.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/1840375144' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:33.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:33 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]: dispatch 2024-09-17T09:20:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: pgmap v323: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]': finished 2024-09-17T09:20:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T09:20:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/1912739202' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.278 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:34.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:34 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: pgmap v323: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:34.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]': finished 2024-09-17T09:20:34.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T09:20:34.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/1912739202' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: pgmap v323: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/3215597613"}]': finished 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/1912739202' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]: dispatch 2024-09-17T09:20:34.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:34.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:34.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:34.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:34 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: pgmap v325: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]': finished 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]: dispatch 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]': finished 2024-09-17T09:20:35.510 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:35.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: pgmap v325: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:35.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]': finished 2024-09-17T09:20:35.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T09:20:35.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:35.512 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.513 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]': finished 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.514 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:20:35.515 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T09:20:35.516 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:20:35.517 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:20:35.518 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.519 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/1647046066' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.520 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[67349]: Cluster is now healthy 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:20:35.521 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.522 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/1647046066' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:20:35.523 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:35 smithi027 ceph-mon[71738]: Cluster is now healthy 2024-09-17T09:20:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: pgmap v325: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6833/210613837"}]': finished 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi.foo.smithi105.mzuaqj"}]': finished 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:20:35.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:20:35.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T09:20:35.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "mon_mds_skip_sanity", "who": "mon"}]': finished 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/1647046066' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-09-17T09:20:35.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:35 smithi105 ceph-mon[70467]: Cluster is now healthy 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all iscsi 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nfs 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:20:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: Upgrade: Finalizing container_image settings 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: Upgrade: Complete! 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]': finished 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/1225481341' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:36.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all iscsi 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nfs 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: Upgrade: Finalizing container_image settings 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: Upgrade: Complete! 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.510 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]': finished 2024-09-17T09:20:36.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T09:20:36.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/1225481341' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:36.511 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:36 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all iscsi 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nfs 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: Upgrade: Finalizing container_image settings 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: Upgrade: Complete! 2024-09-17T09:20:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:20:36.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1132512078"}]': finished 2024-09-17T09:20:36.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T09:20:36.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/1225481341' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:36.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:36 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]: dispatch 2024-09-17T09:20:37.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:37 smithi105 ceph-mon[70467]: pgmap v328: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:37.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:37 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]': finished 2024-09-17T09:20:37.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:37 smithi105 ceph-mon[70467]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T09:20:37.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:37 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/3848205365' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:37.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:37 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:37.478 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[71738]: pgmap v328: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]': finished 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[71738]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/3848205365' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[67349]: pgmap v328: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/610231360"}]': finished 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[67349]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T09:20:37.479 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/3848205365' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:37.480 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:37 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]: dispatch 2024-09-17T09:20:38.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:38 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:38.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:38 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]': finished 2024-09-17T09:20:38.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:38 smithi105 ceph-mon[70467]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T09:20:38.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:38 smithi105 ceph-mon[70467]: from='client.? 172.21.15.105:0/2213497323' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:38.296 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:38 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]': finished 2024-09-17T09:20:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[67349]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T09:20:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[67349]: from='client.? 172.21.15.105:0/2213497323' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:38.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:38.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:0/1126934305"}]': finished 2024-09-17T09:20:38.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[71738]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T09:20:38.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[71738]: from='client.? 172.21.15.105:0/2213497323' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:38.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:38 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]: dispatch 2024-09-17T09:20:39.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[67349]: pgmap v331: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:39.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[67349]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]': finished 2024-09-17T09:20:39.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[67349]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T09:20:39.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[71738]: pgmap v331: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:39.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[71738]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]': finished 2024-09-17T09:20:39.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:39 smithi027 ceph-mon[71738]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T09:20:39.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:39 smithi105 ceph-mon[70467]: pgmap v331: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:39.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:39 smithi105 ceph-mon[70467]: from='client.? ' entity='client.iscsi.foo.smithi105.mzuaqj' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.105:6832/2718211569"}]': finished 2024-09-17T09:20:39.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:39 smithi105 ceph-mon[70467]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T09:20:40.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:40 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:40.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:40 smithi105 ceph-mon[70467]: pgmap v333: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T09:20:40.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:40 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:40.509 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:40 smithi027 ceph-mon[67349]: pgmap v333: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T09:20:40.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:40 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:20:40.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:40 smithi027 ceph-mon[71738]: pgmap v333: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-09-17T09:20:43.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:43 smithi105 ceph-mon[70467]: pgmap v334: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:43.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:43 smithi027 ceph-mon[67349]: pgmap v334: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:43.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:43 smithi027 ceph-mon[71738]: pgmap v334: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:45 smithi027 ceph-mon[67349]: pgmap v335: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:45.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:45 smithi027 ceph-mon[71738]: pgmap v335: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:45.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:45 smithi105 ceph-mon[70467]: pgmap v335: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-09-17T09:20:47.386 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:47 smithi105 ceph-mon[70467]: pgmap v336: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-09-17T09:20:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:47 smithi027 ceph-mon[67349]: pgmap v336: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-09-17T09:20:47.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:47 smithi027 ceph-mon[71738]: pgmap v336: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 588 B/s rd, 0 op/s 2024-09-17T09:20:47.778 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:20:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=sqlstore.transactions t=2024-09-17T09:20:47.385993063Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-17T09:20:47.778 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:20:47 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a[66971]: logger=cleanup t=2024-09-17T09:20:47.388530727Z level=info msg="Completed cleanup jobs" duration=13.016849ms 2024-09-17T09:20:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:48 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:48.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:48 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:48.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:48 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:49 smithi027 ceph-mon[67349]: pgmap v337: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:20:49.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:49 smithi027 ceph-mon[71738]: pgmap v337: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:20:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:49 smithi105 ceph-mon[70467]: pgmap v337: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-09-17T09:20:51.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:51 smithi027 ceph-mon[67349]: pgmap v338: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-17T09:20:51.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:51 smithi027 ceph-mon[71738]: pgmap v338: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-17T09:20:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:51 smithi105 ceph-mon[70467]: pgmap v338: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 875 B/s rd, 0 op/s 2024-09-17T09:20:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:53 smithi027 ceph-mon[67349]: pgmap v339: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:53.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:53 smithi027 ceph-mon[71738]: pgmap v339: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:53.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:53 smithi105 ceph-mon[70467]: pgmap v339: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:54.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:20:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:55 smithi027 ceph-mon[67349]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:55.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:55 smithi027 ceph-mon[67349]: pgmap v340: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:55.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:55 smithi027 ceph-mon[71738]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:55.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:55 smithi027 ceph-mon[71738]: pgmap v340: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:55.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:55 smithi105 ceph-mon[70467]: from='client.54906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:55.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:55 smithi105 ceph-mon[70467]: pgmap v340: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:56.285 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (10m) 73s ago 32m 24.2M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (10m) 25s ago 31m 91.1M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (30s) 25s ago 31m 62.6M - 3.8 980a0bfcd684 7ad9c63573c9 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (11m) 25s ago 35m 439M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (21m) 73s ago 37m 489M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:20:56.286 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (8m) 73s ago 37m 76.4M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:20:56.287 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (9m) 25s ago 36m 62.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:20:56.287 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (7m) 73s ago 36m 54.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:20:56.287 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (11m) 73s ago 32m 16.6M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:20:56.287 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (11m) 25s ago 32m 16.8M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:20:56.287 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (6m) 73s ago 35m 61.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (6m) 73s ago 35m 54.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (5m) 73s ago 35m 48.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (4m) 73s ago 34m 57.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (3m) 25s ago 34m 59.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (3m) 25s ago 34m 57.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:20:56.288 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (2m) 25s ago 33m 56.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:20:56.289 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (2m) 25s ago 33m 54.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:20:56.289 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (10m) 25s ago 33m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:20:56.289 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (82s) 73s ago 31m 76.7M - 18.2.4-855-g19e2a2be 980a0bfcd684 e5c240bc34bc 2024-09-17T09:20:56.289 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (77s) 25s ago 31m 75.1M - 18.2.4-855-g19e2a2be 980a0bfcd684 f99988b09038 2024-09-17T09:20:56.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T09:20:57.251 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:57 smithi027 ceph-mon[67349]: pgmap v341: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:57.251 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:57 smithi027 ceph-mon[71738]: pgmap v341: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:57 smithi105 ceph-mon[70467]: pgmap v341: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:20:58.410 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:58 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:58.410 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:58 smithi027 ceph-mon[67349]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:58.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:58 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:58.410 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:58 smithi027 ceph-mon[71738]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:58 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:20:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:58 smithi105 ceph-mon[70467]: from='client.54912 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:20:59.026 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:20:59.026 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 8 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:20:59.027 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:20:59.028 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 15 2024-09-17T09:20:59.028 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:20:59.028 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:20:59.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:59 smithi027 ceph-mon[67349]: pgmap v342: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:59.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:20:59 smithi027 ceph-mon[67349]: from='client.64864 172.21.15.27:0/146310013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:59.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:59 smithi027 ceph-mon[71738]: pgmap v342: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:59.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:20:59 smithi027 ceph-mon[71738]: from='client.64864 172.21.15.27:0/146310013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:59 smithi105 ceph-mon[70467]: pgmap v342: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:20:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:20:59 smithi105 ceph-mon[70467]: from='client.64864 172.21.15.27:0/146310013' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:20:59.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T09:21:00.225 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:00 smithi027 ceph-mon[67349]: pgmap v343: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:00.226 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:00 smithi027 ceph-mon[71738]: pgmap v343: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:00.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:00 smithi105 ceph-mon[70467]: pgmap v343: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:01.441 INFO:teuthology.orchestra.run.smithi027.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T09:21:02.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'sleep 60' 2024-09-17T09:21:03.410 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:03 smithi027 ceph-mon[67349]: pgmap v344: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:03.411 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:03 smithi027 ceph-mon[71738]: pgmap v344: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:03 smithi105 ceph-mon[70467]: pgmap v344: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:05 smithi027 ceph-mon[67349]: pgmap v345: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:05.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:05 smithi027 ceph-mon[71738]: pgmap v345: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:05 smithi105 ceph-mon[70467]: pgmap v345: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:07.329 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:07 smithi027 ceph-mon[67349]: pgmap v346: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:07.329 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:07 smithi027 ceph-mon[71738]: pgmap v346: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:07 smithi105 ceph-mon[70467]: pgmap v346: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:08.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:08 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:08.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:08 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:08.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:08 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:09.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:09 smithi027 ceph-mon[67349]: pgmap v347: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:09.542 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:09 smithi027 ceph-mon[71738]: pgmap v347: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:09.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:09 smithi105 ceph-mon[70467]: pgmap v347: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:11.330 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:11 smithi105 ceph-mon[70467]: pgmap v348: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:11.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:11 smithi027 ceph-mon[67349]: pgmap v348: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:11.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:11 smithi027 ceph-mon[71738]: pgmap v348: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:13.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:13 smithi027 ceph-mon[67349]: pgmap v349: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:13.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:13 smithi027 ceph-mon[71738]: pgmap v349: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:13.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:13 smithi105 ceph-mon[70467]: pgmap v349: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:15.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:15 smithi027 ceph-mon[67349]: pgmap v350: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:15.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:15 smithi027 ceph-mon[71738]: pgmap v350: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:15.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:15 smithi105 ceph-mon[70467]: pgmap v350: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:17.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:17 smithi027 ceph-mon[67349]: pgmap v351: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:17.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:17 smithi027 ceph-mon[71738]: pgmap v351: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:17 smithi105 ceph-mon[70467]: pgmap v351: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:18.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:18 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:18.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:18 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:18 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:19.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:19 smithi027 ceph-mon[67349]: pgmap v352: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:19.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:19 smithi027 ceph-mon[71738]: pgmap v352: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:19.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:19 smithi105 ceph-mon[70467]: pgmap v352: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:20.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:20 smithi027 ceph-mon[67349]: pgmap v353: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:20.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:20 smithi027 ceph-mon[71738]: pgmap v353: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:20.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:20 smithi105 ceph-mon[70467]: pgmap v353: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:23.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:23 smithi027 ceph-mon[67349]: pgmap v354: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:23.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:23 smithi027 ceph-mon[71738]: pgmap v354: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:23.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:23 smithi105 ceph-mon[70467]: pgmap v354: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:25.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:25 smithi027 ceph-mon[67349]: pgmap v355: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:25.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:25 smithi027 ceph-mon[71738]: pgmap v355: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:25.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:25 smithi105 ceph-mon[70467]: pgmap v355: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:27.310 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:27 smithi027 ceph-mon[71738]: pgmap v356: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:27.311 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:27 smithi027 ceph-mon[67349]: pgmap v356: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:27.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:27 smithi105 ceph-mon[70467]: pgmap v356: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:28.327 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:28 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:28.327 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:28 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:28 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:29.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:29 smithi027 ceph-mon[67349]: pgmap v357: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:29.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:29 smithi027 ceph-mon[71738]: pgmap v357: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:29 smithi105 ceph-mon[70467]: pgmap v357: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:31.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:31 smithi027 ceph-mon[67349]: pgmap v358: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:31.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:31 smithi027 ceph-mon[71738]: pgmap v358: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:31.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:31 smithi105 ceph-mon[70467]: pgmap v358: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:33.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:33 smithi027 ceph-mon[67349]: pgmap v359: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:33.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:33 smithi027 ceph-mon[71738]: pgmap v359: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:33.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:33 smithi105 ceph-mon[70467]: pgmap v359: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[67349]: pgmap v360: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:21:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:21:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:21:35.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[67349]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:21:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[71738]: pgmap v360: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:21:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:21:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:21:35.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:35 smithi027 ceph-mon[71738]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:21:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:35 smithi105 ceph-mon[70467]: pgmap v360: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:21:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:21:35.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:21:35.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:35 smithi105 ceph-mon[70467]: from='mgr.54108 172.21.15.27:0/539498922' entity='mgr.y' 2024-09-17T09:21:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:36 smithi027 ceph-mon[67349]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:21:36.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:36 smithi027 ceph-mon[67349]: pgmap v361: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:36.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:36 smithi027 ceph-mon[71738]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:21:36.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:36 smithi027 ceph-mon[71738]: pgmap v361: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:36 smithi105 ceph-mon[70467]: unable to calc client keyring client.admin placement PlacementSpec(label='_admin'): Cannot place : No matching hosts for label _admin 2024-09-17T09:21:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:36 smithi105 ceph-mon[70467]: pgmap v361: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:37.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:37 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:37.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:37 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:37.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:37 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:38.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:38 smithi027 ceph-mon[67349]: pgmap v362: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:38.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:38 smithi027 ceph-mon[71738]: pgmap v362: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:38.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:38 smithi105 ceph-mon[70467]: pgmap v362: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:41.474 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:41 smithi105 ceph-mon[70467]: pgmap v363: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:41.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:41 smithi027 ceph-mon[71738]: pgmap v363: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:41.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:41 smithi027 ceph-mon[67349]: pgmap v363: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:43.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:43 smithi027 ceph-mon[67349]: pgmap v364: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:43.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:43 smithi027 ceph-mon[71738]: pgmap v364: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:43.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:43 smithi105 ceph-mon[70467]: pgmap v364: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:45.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:45 smithi027 ceph-mon[67349]: pgmap v365: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:45.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:45 smithi027 ceph-mon[71738]: pgmap v365: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:45.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:45 smithi105 ceph-mon[70467]: pgmap v365: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:47.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:47 smithi027 ceph-mon[67349]: pgmap v366: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:47.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:47 smithi027 ceph-mon[71738]: pgmap v366: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:47.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:47 smithi105 ceph-mon[70467]: pgmap v366: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:48.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:48 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:48.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:48 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:48.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:48 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:49.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:49 smithi027 ceph-mon[67349]: pgmap v367: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:49.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:49 smithi027 ceph-mon[71738]: pgmap v367: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:49.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:49 smithi105 ceph-mon[70467]: pgmap v367: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:51.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:51 smithi027 ceph-mon[67349]: pgmap v368: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:51.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:51 smithi027 ceph-mon[71738]: pgmap v368: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:51.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:51 smithi105 ceph-mon[70467]: pgmap v368: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:53.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:53 smithi027 ceph-mon[67349]: pgmap v369: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:53.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:53 smithi027 ceph-mon[71738]: pgmap v369: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:53.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:53 smithi105 ceph-mon[70467]: pgmap v369: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-09-17T09:21:54.434 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:54 smithi027 ceph-mon[71738]: pgmap v370: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:54.434 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:54 smithi027 ceph-mon[67349]: pgmap v370: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:54.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:54 smithi105 ceph-mon[70467]: pgmap v370: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:57.461 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:57 smithi027 ceph-mon[67349]: pgmap v371: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:57.461 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:57 smithi027 ceph-mon[71738]: pgmap v371: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:57.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:57 smithi105 ceph-mon[70467]: pgmap v371: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:21:58.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:58 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:58.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:58 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:58 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:21:59.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:21:59 smithi027 ceph-mon[67349]: pgmap v372: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:59.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:21:59 smithi027 ceph-mon[71738]: pgmap v372: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:21:59.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:21:59 smithi105 ceph-mon[70467]: pgmap v372: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:01.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:01 smithi027 ceph-mon[67349]: pgmap v373: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:01.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:01 smithi027 ceph-mon[71738]: pgmap v373: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:01.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:01 smithi105 ceph-mon[70467]: pgmap v373: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:03.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:03 smithi027 ceph-mon[67349]: pgmap v374: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:03.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:03 smithi027 ceph-mon[71738]: pgmap v374: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:03.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:03 smithi105 ceph-mon[70467]: pgmap v374: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:04.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ps' 2024-09-17T09:22:05.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:05 smithi027 ceph-mon[67349]: pgmap v375: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:05.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:05 smithi027 ceph-mon[71738]: pgmap v375: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:05.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:05 smithi105 ceph-mon[70467]: pgmap v375: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:alertmanager.a smithi027 *:9093,9094 running (11m) 2m ago 33m 24.2M - 0.25.0 c8568f914cd2 92c6277605b3 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:grafana.a smithi105 *:3000 running (11m) 96s ago 33m 91.1M - 9.4.7 954c08fa6188 447ebac1e36e 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:iscsi.foo.smithi105.mzuaqj smithi105 running (101s) 96s ago 32m 62.6M - 3.8 980a0bfcd684 7ad9c63573c9 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.x smithi105 *:8443,8765 running (12m) 96s ago 37m 439M - 18.2.4-855-g19e2a2be 980a0bfcd684 ef416e2a9424 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:mgr.y smithi027 *:8443,8765 running (22m) 2m ago 38m 489M - 18.2.4-855-g19e2a2be 980a0bfcd684 f4c318304d5d 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:mon.a smithi027 running (9m) 2m ago 38m 76.4M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 d855e8be0e46 2024-09-17T09:22:06.519 INFO:teuthology.orchestra.run.smithi027.stdout:mon.b smithi105 running (10m) 96s ago 37m 62.9M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 7b90a4c3691a 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:mon.c smithi027 running (9m) 2m ago 37m 54.3M 2048M 18.2.4-855-g19e2a2be 980a0bfcd684 92f603c1e36f 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.a smithi027 *:9100 running (12m) 2m ago 33m 16.6M - 1.5.0 0da6a335fe13 63779f8bce27 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:node-exporter.b smithi105 *:9100 running (12m) 96s ago 33m 16.8M - 1.5.0 0da6a335fe13 15c01b33b495 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.0 smithi027 running (7m) 2m ago 36m 61.4M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3ef61cb83c37 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.1 smithi027 running (7m) 2m ago 36m 54.2M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 793c309da67a 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.2 smithi027 running (6m) 2m ago 36m 48.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 4d2f7e1c76bb 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.3 smithi027 running (5m) 2m ago 35m 57.9M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 476c2d140d65 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.4 smithi105 running (4m) 96s ago 35m 59.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 aebb444b51e9 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.5 smithi105 running (4m) 96s ago 35m 57.0M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 c28b73a112c5 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.6 smithi105 running (3m) 96s ago 35m 56.5M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 bd0da36c5e64 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:osd.7 smithi105 running (3m) 96s ago 34m 54.7M 4096M 18.2.4-855-g19e2a2be 980a0bfcd684 3bb4e3186543 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:prometheus.a smithi105 *:9095 running (11m) 96s ago 34m 55.9M - 2.43.0 a07b618ecd1d f7d88b100074 2024-09-17T09:22:06.520 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi027.yrcgts smithi027 *:8000 running (2m) 2m ago 32m 76.7M - 18.2.4-855-g19e2a2be 980a0bfcd684 e5c240bc34bc 2024-09-17T09:22:06.521 INFO:teuthology.orchestra.run.smithi027.stdout:rgw.foo.smithi105.afiseg smithi105 *:8000 running (2m) 96s ago 32m 75.1M - 18.2.4-855-g19e2a2be 980a0bfcd684 f99988b09038 2024-09-17T09:22:07.116 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:07 smithi027 ceph-mon[67349]: pgmap v376: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:07.116 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:07 smithi027 ceph-mon[71738]: pgmap v376: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:07.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions' 2024-09-17T09:22:07.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:07 smithi105 ceph-mon[70467]: pgmap v376: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:08.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:08.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[67349]: from='client.64870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:08.406 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[67349]: pgmap v377: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:08.406 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:08.406 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[71738]: from='client.64870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:08.407 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:08 smithi027 ceph-mon[71738]: pgmap v377: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:08.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:08 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:08.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:08 smithi105 ceph-mon[70467]: from='client.64870 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:08.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:08 smithi105 ceph-mon[70467]: pgmap v377: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:09.189 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "mon": { 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 3 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "mgr": { 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "osd": { 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 8 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "rgw": { 2024-09-17T09:22:09.190 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 2 2024-09-17T09:22:09.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2024-09-17T09:22:09.191 INFO:teuthology.orchestra.run.smithi027.stdout: "overall": { 2024-09-17T09:22:09.191 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)": 15 2024-09-17T09:22:09.191 INFO:teuthology.orchestra.run.smithi027.stdout: } 2024-09-17T09:22:09.191 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:22:09.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:09 smithi027 ceph-mon[67349]: from='client.64876 172.21.15.27:0/2940157745' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:09.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:09 smithi027 ceph-mon[71738]: from='client.64876 172.21.15.27:0/2940157745' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:09.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:09 smithi105 ceph-mon[70467]: from='client.64876 172.21.15.27:0/2940157745' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:09.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:22:10.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:10 smithi027 ceph-mon[67349]: pgmap v378: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:10.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:10 smithi027 ceph-mon[71738]: pgmap v378: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:10.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:10 smithi105 ceph-mon[70467]: pgmap v378: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:12.001 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "target_image": null, 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "in_progress": false, 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "which": "", 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "services_complete": [], 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "progress": null, 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "message": "", 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout: "is_paused": false 2024-09-17T09:22:12.002 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:22:12.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph health detail' 2024-09-17T09:22:13.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:13 smithi027 ceph-mon[67349]: from='client.64882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:13.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:13 smithi027 ceph-mon[67349]: pgmap v379: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:13 smithi027 ceph-mon[71738]: from='client.64882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:13.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:13 smithi027 ceph-mon[71738]: pgmap v379: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:13.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:13 smithi105 ceph-mon[70467]: from='client.64882 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:13.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:13 smithi105 ceph-mon[70467]: pgmap v379: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:14.698 INFO:teuthology.orchestra.run.smithi027.stdout:HEALTH_OK 2024-09-17T09:22:15.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:15 smithi027 ceph-mon[67349]: pgmap v380: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:15.368 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:15 smithi027 ceph-mon[67349]: from='client.? 172.21.15.27:0/2942099320' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:22:15.368 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:15 smithi027 ceph-mon[71738]: pgmap v380: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:15.369 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:15 smithi027 ceph-mon[71738]: from='client.? 172.21.15.27:0/2942099320' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:22:15.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T09:22:15.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:15 smithi105 ceph-mon[70467]: pgmap v380: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:15.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:15 smithi105 ceph-mon[70467]: from='client.? 172.21.15.27:0/2942099320' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:22:17.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:17 smithi027 ceph-mon[67349]: pgmap v381: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:17.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:17 smithi027 ceph-mon[71738]: pgmap v381: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:17.430 INFO:teuthology.orchestra.run.smithi027.stdout:true 2024-09-17T09:22:17.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:17 smithi105 ceph-mon[70467]: pgmap v381: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:18.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T09:22:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:18 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:18.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:18 smithi027 ceph-mon[67349]: from='client.64894 172.21.15.27:0/1255276777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:18.258 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:18 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:18.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:18 smithi027 ceph-mon[71738]: from='client.64894 172.21.15.27:0/1255276777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:18 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:18.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:18 smithi105 ceph-mon[70467]: from='client.64894 172.21.15.27:0/1255276777' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:19.367 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:19 smithi027 ceph-mon[67349]: pgmap v382: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:19.368 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:19 smithi027 ceph-mon[71738]: pgmap v382: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:19.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:19 smithi105 ceph-mon[70467]: pgmap v382: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:20.089 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph version 18.2.4-855-g19e2a2be (19e2a2beb6f41a3ab62650e38614eb29b59dcbb9) reef (stable)" 2024-09-17T09:22:20.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:20 smithi027 ceph-mon[67349]: pgmap v383: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:20.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:20 smithi027 ceph-mon[67349]: from='client.64900 172.21.15.27:0/2468337293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:20.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:20 smithi027 ceph-mon[71738]: pgmap v383: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:20.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:20 smithi027 ceph-mon[71738]: from='client.64900 172.21.15.27:0/2468337293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:20.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:20 smithi105 ceph-mon[70467]: pgmap v383: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:20.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:20 smithi105 ceph-mon[70467]: from='client.64900 172.21.15.27:0/2468337293' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:22:20.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -e sha1=19e2a2beb6f41a3ab62650e38614eb29b59dcbb9 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-09-17T09:22:22.757 INFO:teuthology.orchestra.run.smithi027.stdout:osd 8 2m ago - 2024-09-17T09:22:23.396 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:23 smithi027 ceph-mon[67349]: pgmap v384: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:23.396 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:23 smithi027 ceph-mon[71738]: pgmap v384: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:23.422 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:22:23.432 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi027.front.sepia.ceph.com 2024-09-17T09:22:23.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-09-17T09:22:23.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:23 smithi105 ceph-mon[70467]: pgmap v384: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:24.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:24 smithi027 ceph-mon[67349]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:24.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:24 smithi027 ceph-mon[71738]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:24.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:24 smithi105 ceph-mon[70467]: from='client.64906 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:25.508 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:25 smithi027 ceph-mon[67349]: pgmap v385: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:25.509 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:25 smithi027 ceph-mon[71738]: pgmap v385: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:25.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:25 smithi105 ceph-mon[70467]: pgmap v385: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:25.761 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "image": "quay.io/ceph/ceph", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "registry": "quay.io", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "bare_image": "ceph/ceph", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "versions": [ 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "19.1.1", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "19.1.0", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "18.2.4", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "18.2.2", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "18.2.1", 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: "18.2.0" 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2024-09-17T09:22:25.762 INFO:teuthology.orchestra.run.smithi027.stdout:} 2024-09-17T09:22:26.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-09-17T09:22:27.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:27 smithi027 ceph-mon[71738]: from='client.64912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:27.359 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:27 smithi027 ceph-mon[71738]: pgmap v386: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:27.360 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:27 smithi027 ceph-mon[67349]: from='client.64912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:27.360 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:27 smithi027 ceph-mon[67349]: pgmap v386: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:27.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:27 smithi105 ceph-mon[70467]: from='client.64912 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:27.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:27 smithi105 ceph-mon[70467]: pgmap v386: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:28 smithi027 ceph-mon[67349]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:28.258 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:28 smithi027 ceph-mon[67349]: pgmap v387: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:28 smithi027 ceph-mon[71738]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:28.259 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:28 smithi027 ceph-mon[71738]: pgmap v387: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:28 smithi105 ceph-mon[70467]: from='client.44867 -' entity='client.iscsi.foo.smithi105.mzuaqj' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-09-17T09:22:28.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:28 smithi105 ceph-mon[70467]: pgmap v387: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-09-17T09:22:28.765 INFO:teuthology.orchestra.run.smithi027.stdout: "16.2.0", 2024-09-17T09:22:29.459 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:29 smithi027 ceph-mon[71738]: from='client.64918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:29.459 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:29 smithi027 ceph-mon[67349]: from='client.64918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:29.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v16.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-09-17T09:22:29.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:29 smithi105 ceph-mon[70467]: from='client.64918 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:22:30.450 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:30 smithi027 ceph-mon[67349]: pgmap v388: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:30.450 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:30 smithi027 ceph-mon[71738]: pgmap v388: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:30.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:30 smithi105 ceph-mon[70467]: pgmap v388: 161 pgs: 161 active+clean; 604 KiB data, 317 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-09-17T09:22:31.878 INFO:teuthology.orchestra.run.smithi027.stdout: "v16.2.2", 2024-09-17T09:22:31.879 INFO:teuthology.orchestra.run.smithi027.stdout: "v16.2.2-20210505", 2024-09-17T09:22:32.551 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T09:22:32.561 INFO:tasks.cephadm:Teardown begin 2024-09-17T09:22:32.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:22:32.592 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:22:32.626 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T09:22:32.627 DEBUG:teuthology.orchestra.run.smithi027:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T09:22:32.649 DEBUG:teuthology.orchestra.run.smithi105:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T09:22:32.684 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T09:22:32.685 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-17T09:22:32.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a 2024-09-17T09:22:33.008 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:32 smithi027 systemd[1]: Stopping Ceph mon.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:33.345 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a[67345]: 2024-09-17T09:22:33.018+0000 7fd66d3b6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:33.345 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:33 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a[67345]: 2024-09-17T09:22:33.018+0000 7fd66d3b6640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-17T09:22:33.345 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:33 smithi027 podman[110292]: 2024-09-17 09:22:33.070759575 +0000 UTC m=+0.177488709 container died d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:22:33.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:33 smithi027 podman[110292]: 2024-09-17 09:22:33.460914352 +0000 UTC m=+0.567643468 container cleanup d855e8be0e46b252375afd0ab8999699160eece12677ec22688cd455b4ba92a6 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD) 2024-09-17T09:22:33.663 INFO:journalctl@ceph.mon.a.smithi027.stdout:Sep 17 09:22:33 smithi027 bash[110292]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-a 2024-09-17T09:22:33.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.a.service' 2024-09-17T09:22:33.867 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:33.868 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-17T09:22:33.868 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-17T09:22:33.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.c 2024-09-17T09:22:34.166 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:33 smithi027 systemd[1]: Stopping Ceph mon.c for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:34.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c[71734]: 2024-09-17T09:22:34.165+0000 7efeaf2bd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:34.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:34 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c[71734]: 2024-09-17T09:22:34.165+0000 7efeaf2bd640 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-09-17T09:22:34.508 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:34 smithi027 podman[110416]: 2024-09-17 09:22:34.34934501 +0000 UTC m=+0.288210564 container died 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T09:22:34.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:34 smithi027 podman[110416]: 2024-09-17 09:22:34.698848964 +0000 UTC m=+0.637714510 container cleanup 92f603c1e36fdacd7a3a256ca02231427a6c536c24eb2cf08e38ccb7e9f9165c (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T09:22:34.909 INFO:journalctl@ceph.mon.c.smithi027.stdout:Sep 17 09:22:34 smithi027 bash[110416]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-c 2024-09-17T09:22:35.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.c.service' 2024-09-17T09:22:35.163 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:35.163 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-17T09:22:35.163 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-17T09:22:35.163 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.b 2024-09-17T09:22:35.481 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:35 smithi105 systemd[1]: Stopping Ceph mon.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:35.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b[70463]: 2024-09-17T09:22:35.480+0000 7f7cb2d26640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:35.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b[70463]: 2024-09-17T09:22:35.480+0000 7f7cb2d26640 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-09-17T09:22:35.778 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:35 smithi105 podman[109195]: 2024-09-17 09:22:35.715298371 +0000 UTC m=+0.346843073 container died 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux ) 2024-09-17T09:22:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:36 smithi105 podman[109195]: 2024-09-17 09:22:36.188732474 +0000 UTC m=+0.820277215 container cleanup 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:22:36.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:36 smithi105 bash[109195]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b 2024-09-17T09:22:36.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 17 09:22:36 smithi105 podman[109207]: 2024-09-17 09:22:36.490324877 +0000 UTC m=+0.770996271 container remove 7b90a4c3691a40ae32288a31febdba33f4cbe6fb02d419da19e997b3bd38994b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mon-b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:22:36.622 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mon.b.service' 2024-09-17T09:22:36.668 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:36.668 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-17T09:22:36.668 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-17T09:22:36.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y 2024-09-17T09:22:37.008 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:36 smithi027 systemd[1]: Stopping Ceph mgr.y for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:37.455 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:22:37.120+0000 7f5b29af4640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:37.455 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:37 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y[52509]: 2024-09-17T09:22:37.120+0000 7f5b29af4640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-17T09:22:37.456 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:37 smithi027 podman[110538]: 2024-09-17 09:22:37.162609106 +0000 UTC m=+0.300047093 container died f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T09:22:37.758 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:37 smithi027 podman[110538]: 2024-09-17 09:22:37.554575947 +0000 UTC m=+0.692013927 container cleanup f4c318304d5d3c4675296ed7d1821014c7f9b16925431672f38b8bd55bda65e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T09:22:37.758 INFO:journalctl@ceph.mgr.y.smithi027.stdout:Sep 17 09:22:37 smithi027 bash[110538]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-y 2024-09-17T09:22:37.988 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-17T09:22:38.024 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:38.024 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-17T09:22:38.024 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-17T09:22:38.024 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x 2024-09-17T09:22:38.384 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:22:38 smithi105 systemd[1]: Stopping Ceph mgr.x for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:38.662 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:22:38 smithi105 podman[109320]: 2024-09-17 09:22:38.382835373 +0000 UTC m=+0.164062645 container died ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T09:22:39.013 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:22:38 smithi105 podman[109320]: 2024-09-17 09:22:38.810516527 +0000 UTC m=+0.591743844 container cleanup ef416e2a9424335a495fc1b2881ea769a96457b07cb3a7566bb391f06e176714 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T09:22:39.013 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 17 09:22:38 smithi105 bash[109320]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-mgr-x 2024-09-17T09:22:39.237 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-17T09:22:39.272 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:39.272 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-17T09:22:39.272 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T09:22:39.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.0 2024-09-17T09:22:39.601 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:39 smithi027 systemd[1]: Stopping Ceph osd.0 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:40.008 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:22:39.600+0000 7f8700bd9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:40.008 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:22:39.600+0000 7f8700bd9640 -1 osd.0 130 *** Got signal Terminated *** 2024-09-17T09:22:40.008 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:39 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0[78579]: 2024-09-17T09:22:39.600+0000 7f8700bd9640 -1 osd.0 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:22:45.258 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:44 smithi027 podman[110659]: 2024-09-17 09:22:44.976677824 +0000 UTC m=+5.510208675 container died 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:22:45.594 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:45 smithi027 podman[110659]: 2024-09-17 09:22:45.359112051 +0000 UTC m=+5.892642893 container cleanup 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-19e2a2b) 2024-09-17T09:22:45.594 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:45 smithi027 bash[110659]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0 2024-09-17T09:22:45.939 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:45 smithi027 podman[110674]: 2024-09-17 09:22:45.593342698 +0000 UTC m=+0.609997105 container remove 3ef61cb83c37064ea1b243a659588d20a6e289d2717780c5dbaa54ecbec9054d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:22:46.195 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:45 smithi027 podman[110763]: 2024-09-17 09:22:45.841480226 +0000 UTC m=+0.016656232 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:22:46.195 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:45 smithi027 podman[110763]: 2024-09-17 09:22:45.944278561 +0000 UTC m=+0.119454599 container create 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T09:22:46.484 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:46 smithi027 podman[110763]: 2024-09-17 09:22:46.288509348 +0000 UTC m=+0.463685347 container init 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-17T09:22:46.485 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:46 smithi027 podman[110763]: 2024-09-17 09:22:46.292441721 +0000 UTC m=+0.467617721 container start 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:22:46.485 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:46 smithi027 podman[110763]: 2024-09-17 09:22:46.371770116 +0000 UTC m=+0.546946127 container attach 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T09:22:46.758 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:46 smithi027 podman[110763]: 2024-09-17 09:22:46.483814428 +0000 UTC m=+0.658990428 container died 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T09:22:47.088 INFO:journalctl@ceph.osd.0.smithi027.stdout:Sep 17 09:22:47 smithi027 podman[110787]: 2024-09-17 09:22:47.083980003 +0000 UTC m=+0.588266883 container remove 3603551ca20ae7cd3bf362ada6aabe6ee3ef1b085ce07e777013bd2b119d69dd (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T09:22:47.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T09:22:47.146 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:47.147 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T09:22:47.147 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T09:22:47.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.1 2024-09-17T09:22:47.357 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:47 smithi027 systemd[1]: Stopping Ceph osd.1 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:22:47.520+0000 7fb0f6754640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:22:47.520+0000 7fb0f6754640 -1 osd.1 130 *** Got signal Terminated *** 2024-09-17T09:22:47.758 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:47 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1[83482]: 2024-09-17T09:22:47.520+0000 7fb0f6754640 -1 osd.1 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:22:53.258 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:52 smithi027 podman[110868]: 2024-09-17 09:22:52.927051366 +0000 UTC m=+5.523732881 container died 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True) 2024-09-17T09:22:53.630 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:53 smithi027 podman[110868]: 2024-09-17 09:22:53.445600682 +0000 UTC m=+6.042282191 container cleanup 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:22:53.630 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:53 smithi027 bash[110868]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1 2024-09-17T09:22:54.008 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:53 smithi027 podman[110896]: 2024-09-17 09:22:53.629948829 +0000 UTC m=+0.699743720 container remove 793c309da67a50b16a1820506e995dfd5ebf3f407074242cf0979d47b167748b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T09:22:54.320 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:53.955577039 +0000 UTC m=+0.016236650 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:22:54.320 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:54.198107621 +0000 UTC m=+0.258767268 container create a89d78443cf6fb8fba26892ae22d3e4f2a686a13e1469c82f8fbd457857f91e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T09:22:54.635 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:54.433869041 +0000 UTC m=+0.494528649 container init a89d78443cf6fb8fba26892ae22d3e4f2a686a13e1469c82f8fbd457857f91e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T09:22:54.635 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:54.437298364 +0000 UTC m=+0.497957966 container start a89d78443cf6fb8fba26892ae22d3e4f2a686a13e1469c82f8fbd457857f91e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T09:22:54.636 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:54.542233615 +0000 UTC m=+0.602893217 container attach a89d78443cf6fb8fba26892ae22d3e4f2a686a13e1469c82f8fbd457857f91e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T09:22:54.929 INFO:journalctl@ceph.osd.1.smithi027.stdout:Sep 17 09:22:54 smithi027 podman[110982]: 2024-09-17 09:22:54.635302819 +0000 UTC m=+0.695962431 container died a89d78443cf6fb8fba26892ae22d3e4f2a686a13e1469c82f8fbd457857f91e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T09:22:55.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T09:22:55.243 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:22:55.243 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T09:22:55.243 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T09:22:55.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.2 2024-09-17T09:22:55.508 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:22:55 smithi027 systemd[1]: Stopping Ceph osd.2 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:22:56.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:22:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:22:55.597+0000 7f3bc652c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:22:56.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:22:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:22:55.597+0000 7f3bc652c640 -1 osd.2 130 *** Got signal Terminated *** 2024-09-17T09:22:56.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:22:55 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2[89821]: 2024-09-17T09:22:55.597+0000 7f3bc652c640 -1 osd.2 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:01.392 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:01 smithi027 podman[111086]: 2024-09-17 09:23:01.117965358 +0000 UTC m=+5.652502221 container died 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T09:23:01.667 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:01 smithi027 podman[111086]: 2024-09-17 09:23:01.465405405 +0000 UTC m=+5.999942265 container cleanup 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T09:23:01.667 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:01 smithi027 bash[111086]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2 2024-09-17T09:23:02.008 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:01 smithi027 podman[111106]: 2024-09-17 09:23:01.666678907 +0000 UTC m=+0.543237544 container remove 4d2f7e1c76bb967dca8fb876e3d6525813010ff84ed5a60d30e739d397ac9e35 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T09:23:02.349 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.00067072 +0000 UTC m=+0.017104847 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:02.349 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.109231517 +0000 UTC m=+0.125665661 container create 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-17T09:23:02.349 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.345004275 +0000 UTC m=+0.361438374 container init 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T09:23:02.349 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.348682036 +0000 UTC m=+0.365116137 container start 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:23:02.758 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.444983248 +0000 UTC m=+0.461417355 container attach 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:23:02.758 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:02 smithi027 podman[111194]: 2024-09-17 09:23:02.549100021 +0000 UTC m=+0.565534130 container died 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T09:23:03.106 INFO:journalctl@ceph.osd.2.smithi027.stdout:Sep 17 09:23:03 smithi027 podman[111217]: 2024-09-17 09:23:03.090110032 +0000 UTC m=+0.535099222 container remove 7571495ba5911a7c775535e99f3c3cceefd18b374970fd1b83c64dc8ad1faa45 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:23:03.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T09:23:03.153 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:03.153 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T09:23:03.153 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T09:23:03.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3 2024-09-17T09:23:03.485 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:03 smithi027 systemd[1]: Stopping Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:03.758 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:23:03.483+0000 7ff8eecbc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:23:03.758 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:23:03.483+0000 7ff8eecbc640 -1 osd.3 130 *** Got signal Terminated *** 2024-09-17T09:23:03.758 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:03 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:23:03.483+0000 7ff8eecbc640 -1 osd.3 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:07.279 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:06.803+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:07.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:06 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:06.909+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:07.758 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:07 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:23:07.371+0000 7ff8e72c8640 -1 osd.3 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.428172+0000 front 2024-09-17T09:22:42.428120+0000 (oldest deadline 2024-09-17T09:23:06.528160+0000) 2024-09-17T09:23:08.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:07.864+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:08.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:07 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:07.769+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:08.758 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:08 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3[96271]: 2024-09-17T09:23:08.383+0000 7ff8e72c8640 -1 osd.3 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.428172+0000 front 2024-09-17T09:22:42.428120+0000 (oldest deadline 2024-09-17T09:23:06.528160+0000) 2024-09-17T09:23:09.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:08.763+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:09.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:08 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:08.888+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:09.096 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:08 smithi027 podman[111297]: 2024-09-17 09:23:08.813619471 +0000 UTC m=+5.435024174 container died 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:23:09.453 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111297]: 2024-09-17 09:23:09.19288678 +0000 UTC m=+5.814291476 container cleanup 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T09:23:09.453 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 bash[111297]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3 2024-09-17T09:23:09.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:09.062+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:09.753 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111315]: 2024-09-17 09:23:09.45224169 +0000 UTC m=+0.635746218 container remove 476c2d140d65cfc86ebc50df243c3759425e4f1026a91914229750143c878774 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T09:23:10.008 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111402]: 2024-09-17 09:23:09.752985764 +0000 UTC m=+0.093725990 container create e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:23:10.008 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111402]: 2024-09-17 09:23:09.675595818 +0000 UTC m=+0.016336044 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:10.009 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111402]: 2024-09-17 09:23:09.964037818 +0000 UTC m=+0.304778061 container init e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T09:23:10.009 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:09 smithi027 podman[111402]: 2024-09-17 09:23:09.967324527 +0000 UTC m=+0.308064756 container start e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T09:23:10.028 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:09.892+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:10.029 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:09.757+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:10.029 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:09 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:09.785+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:10.459 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 podman[111402]: 2024-09-17 09:23:10.055787968 +0000 UTC m=+0.396528198 container attach e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:23:10.459 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 podman[111402]: 2024-09-17 09:23:10.159282345 +0000 UTC m=+0.500022578 container died e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T09:23:10.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:10.048+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:10.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T09:23:10.730 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 podman[111425]: 2024-09-17 09:23:10.683931997 +0000 UTC m=+0.514303368 container remove e4718fb6fd45fe9a068603e8d1dee64822bbeba55f7214103a5b2166079ef8e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T09:23:10.731 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T09:23:10.731 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 systemd[1]: Stopped Ceph osd.3 for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:23:10.731 INFO:journalctl@ceph.osd.3.smithi027.stdout:Sep 17 09:23:10 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.3.service: Consumed 6.299s CPU time. 2024-09-17T09:23:10.746 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:10.746 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T09:23:10.746 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T09:23:10.746 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.4 2024-09-17T09:23:11.028 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:10 smithi105 systemd[1]: Stopping Ceph osd.4 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:11.029 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:10.815+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:11.029 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:10.799+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:11.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:10 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:10.936+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:11.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:11.020+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:11.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:11.051+0000 7fb0f0e52640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:23:11.528 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:11.051+0000 7fb0f0e52640 -1 osd.4 130 *** Got signal Terminated *** 2024-09-17T09:23:11.529 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:11.051+0000 7fb0f0e52640 -1 osd.4 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:12.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:11.865+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:12.279 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:11.811+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:12.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:11.983+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:12.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:11 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:11.992+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:13.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:12.893+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:13.279 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:12.837+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:13.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:13.024+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:13.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:12 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:12.993+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:14.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:13.856+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:14.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:13.840+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:14.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:13 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:13.961+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:14.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:14.045+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:14.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:14.045+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:15.263 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:14.872+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:15.263 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:14.872+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:49.254065+0000 front 2024-09-17T09:22:49.254093+0000 (oldest deadline 2024-09-17T09:23:14.554069+0000) 2024-09-17T09:23:15.263 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:14.826+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:15.263 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:14.826+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:15.264 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:14.999+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:15.264 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:14.999+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:15.264 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:14.943+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:15.264 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:14 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:14.943+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:15.528 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:15.262Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-09-17T09:23:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:15.265Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-09-17T09:23:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:15.265Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-09-17T09:23:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:15.265Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-09-17T09:23:15.529 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:15.266Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.27:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.27:8765: connect: connection refused" 2024-09-17T09:23:16.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:15.820+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:16.279 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:15.820+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:16.279 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:15.902+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:43.953650+0000 front 2024-09-17T09:22:43.953802+0000 (oldest deadline 2024-09-17T09:23:09.253677+0000) 2024-09-17T09:23:16.279 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4[81811]: 2024-09-17T09:23:15.903+0000 7fb0e945e640 -1 osd.4 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:49.254065+0000 front 2024-09-17T09:22:49.254093+0000 (oldest deadline 2024-09-17T09:23:14.554069+0000) 2024-09-17T09:23:16.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:15.939+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:16.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:15.939+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:16.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:15.993+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:16.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:15 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:15.993+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:16.690 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:16 smithi105 podman[109456]: 2024-09-17 09:23:16.366196223 +0000 UTC m=+5.414746426 container died aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD) 2024-09-17T09:23:16.970 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:16.783+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:16.971 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:16.783+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:16.971 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:16 smithi105 podman[109456]: 2024-09-17 09:23:16.871474183 +0000 UTC m=+5.920024448 container cleanup aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:23:16.971 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:16 smithi105 bash[109456]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4 2024-09-17T09:23:17.278 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109470]: 2024-09-17 09:23:17.081062665 +0000 UTC m=+0.709280694 container remove aebb444b51e9a15a5117bf367f96a67e3837e48d6f8c969cab57b69afc43d0a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b) 2024-09-17T09:23:17.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:16.983+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:17.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:16.983+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:17.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:16.969+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:17.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:16 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:16.969+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:17.550 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.318659407 +0000 UTC m=+0.016152583 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:17.892 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.440982869 +0000 UTC m=+0.138476029 container create a57029f56aa2f5f1c7a2b435f17c30606c258d36ba6c71e7ff15372ee10ba17b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T09:23:17.893 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.680725457 +0000 UTC m=+0.378218627 container init a57029f56aa2f5f1c7a2b435f17c30606c258d36ba6c71e7ff15372ee10ba17b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b) 2024-09-17T09:23:17.893 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.683931213 +0000 UTC m=+0.381424375 container start a57029f56aa2f5f1c7a2b435f17c30606c258d36ba6c71e7ff15372ee10ba17b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T09:23:17.893 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.760153767 +0000 UTC m=+0.457646932 container attach a57029f56aa2f5f1c7a2b435f17c30606c258d36ba6c71e7ff15372ee10ba17b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T09:23:18.156 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:17.813+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:18.156 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:17.813+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:18.156 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:17.978+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:18.156 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:17.978+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:18.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:17.957+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:18.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:17 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:17.957+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:18.157 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 17 09:23:17 smithi105 podman[109559]: 2024-09-17 09:23:17.874761826 +0000 UTC m=+0.572254987 container died a57029f56aa2f5f1c7a2b435f17c30606c258d36ba6c71e7ff15372ee10ba17b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=reef-19e2a2b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:23:18.459 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T09:23:18.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:18.499 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T09:23:18.500 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T09:23:18.500 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.5 2024-09-17T09:23:18.699 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 systemd[1]: Stopping Ceph osd.5 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:18.977 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:18.780+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:18.977 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:18.780+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:18.977 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:18.833+0000 7f6c2ae9a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:23:18.977 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:18.833+0000 7f6c2ae9a640 -1 osd.5 130 *** Got signal Terminated *** 2024-09-17T09:23:18.977 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:18.833+0000 7f6c2ae9a640 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:19.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:18.986+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:19.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:18.986+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:19.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:18.975+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:19.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:18 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:18.975+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:19.943 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:19.752+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:19.943 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:19.752+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:20.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:19.982+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:20.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:19.982+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:20.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:19.942+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:20.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:19 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:19.942+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:21.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:20.777+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:21.029 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:20.777+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:21.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:20.892+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:21.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:20.892+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:21.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:20.943+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:21.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:20 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:20.943+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:22.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:21.753+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:22.028 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:21.753+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:22.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:21.927+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:22.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:21.927+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:22.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:21.985+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:22.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:21 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:21.985+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:22.986 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:22.710+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:22.987 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:22.710+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:22.987 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:22.900+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:22.987 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:22.900+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:23.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:22.985+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:23.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:22 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:22.985+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:23.740+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:40.665520+0000 front 2024-09-17T09:22:40.665649+0000 (oldest deadline 2024-09-17T09:23:06.565474+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5[86646]: 2024-09-17T09:23:23.740+0000 7f6c234a6640 -1 osd.5 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.066706+0000 front 2024-09-17T09:22:50.066682+0000 (oldest deadline 2024-09-17T09:23:14.166626+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:23.916+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:23.916+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:23.916+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:23.978+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:24.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:23.978+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:24.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:23 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:23.978+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:24.527 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:24 smithi105 podman[109662]: 2024-09-17 09:23:24.275787523 +0000 UTC m=+5.538656630 container died c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T09:23:24.778 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:24 smithi105 podman[109662]: 2024-09-17 09:23:24.708967201 +0000 UTC m=+5.971836302 container cleanup c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2) 2024-09-17T09:23:24.778 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:24 smithi105 bash[109662]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5 2024-09-17T09:23:25.278 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:24 smithi105 podman[109690]: 2024-09-17 09:23:24.885507413 +0000 UTC m=+0.607157220 container remove c28b73a112c5e9144cddda32690b6d4fc2048d8b401a7a6df6a1f1e9178b261d (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:24.984+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:24.984+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:24.984+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:24.938+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:24.938+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:25.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:24 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:24.938+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:25.597 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.319791831 +0000 UTC m=+0.110671190 container create 2bd76fc4b0f55e705afd17a09d4028eeb198bda69e7f58a1c1faf12a3761e69f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T09:23:25.598 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.224809999 +0000 UTC m=+0.015689357 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:25.598 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.522951617 +0000 UTC m=+0.313830977 container init 2bd76fc4b0f55e705afd17a09d4028eeb198bda69e7f58a1c1faf12a3761e69f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:23:25.598 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.52693558 +0000 UTC m=+0.317814943 container start 2bd76fc4b0f55e705afd17a09d4028eeb198bda69e7f58a1c1faf12a3761e69f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:23:25.598 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.597174366 +0000 UTC m=+0.388053736 container attach 2bd76fc4b0f55e705afd17a09d4028eeb198bda69e7f58a1c1faf12a3761e69f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0) 2024-09-17T09:23:25.928 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 17 09:23:25 smithi105 podman[109782]: 2024-09-17 09:23:25.712416037 +0000 UTC m=+0.503295401 container died 2bd76fc4b0f55e705afd17a09d4028eeb198bda69e7f58a1c1faf12a3761e69f (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T09:23:25.928 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:25.926+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:25.929 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:25.926+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:25.929 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:25.926+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:26.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:25.982+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:26.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:25.982+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:26.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:25 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:25.982+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:26.361 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T09:23:26.399 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:26.400 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T09:23:26.400 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T09:23:26.400 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.6 2024-09-17T09:23:26.722 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 systemd[1]: Stopping Ceph osd.6 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:27.028 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:26.937+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:26.937+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:26.937+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.720+0000 7fd62ad05640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.720+0000 7fd62ad05640 -1 osd.6 130 *** Got signal Terminated *** 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.720+0000 7fd62ad05640 -1 osd.6 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.952+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.952+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:27.029 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:26 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:26.952+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:28.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:27.957+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:28.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:27.957+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:28.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:27.957+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:28.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:27.960+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:28.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:27.960+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:28.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:27 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:27.960+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:29.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:28.988+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:29.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:28.988+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:29.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:28.988+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:29.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:28.966+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:29.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:28.966+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:29.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:28 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:28.966+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:30.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:29.967+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:30.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:29.967+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:30.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:29.967+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:30.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:29.979+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:30.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:29.979+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:30.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:29 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:29.979+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:31.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:31.014+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:31.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:31.014+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:31.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:31 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:31.014+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:31.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:30.972+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.577115+0000 front 2024-09-17T09:22:42.576992+0000 (oldest deadline 2024-09-17T09:23:06.076976+0000) 2024-09-17T09:23:31.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:30.972+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.777776+0000 front 2024-09-17T09:22:50.777871+0000 (oldest deadline 2024-09-17T09:23:13.677896+0000) 2024-09-17T09:23:31.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:30.972+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:57.778745+0000 front 2024-09-17T09:22:57.778769+0000 (oldest deadline 2024-09-17T09:23:23.078652+0000) 2024-09-17T09:23:31.279 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:30 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6[91211]: 2024-09-17T09:23:30.972+0000 7fd623311640 -1 osd.6 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:08.281173+0000 front 2024-09-17T09:23:08.281220+0000 (oldest deadline 2024-09-17T09:23:30.581103+0000) 2024-09-17T09:23:32.278 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:32 smithi105 podman[109887]: 2024-09-17 09:23:32.190295358 +0000 UTC m=+5.576581954 container died bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-19e2a2b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:23:32.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:32.015+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:32.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:32.015+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:32.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:32 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:32.015+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:32.779 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:32 smithi105 podman[109887]: 2024-09-17 09:23:32.587153273 +0000 UTC m=+5.973439900 container cleanup bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:23:32.779 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:32 smithi105 bash[109887]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6 2024-09-17T09:23:33.134 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:32 smithi105 podman[109903]: 2024-09-17 09:23:32.813163591 +0000 UTC m=+0.612311666 container remove bd0da36c5e645db5047b64fbad73b552a981e97fe5e0783afd1f7202bbbaf36b (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2) 2024-09-17T09:23:33.135 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:33.011+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:33.135 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:33.011+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:33.135 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:33 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:33.012+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:33.503 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.036410961 +0000 UTC m=+0.016314078 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:33.503 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.158045971 +0000 UTC m=+0.137949120 container create 48c3d9e795cd465bd80f97aafe0f8f3503eda24622cf74ebf433d1f6fae23ba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T09:23:33.503 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.377348932 +0000 UTC m=+0.357252046 container init 48c3d9e795cd465bd80f97aafe0f8f3503eda24622cf74ebf433d1f6fae23ba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T09:23:33.503 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.380755575 +0000 UTC m=+0.360658682 container start 48c3d9e795cd465bd80f97aafe0f8f3503eda24622cf74ebf433d1f6fae23ba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-19e2a2b, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:23:33.503 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.502138944 +0000 UTC m=+0.482042062 container attach 48c3d9e795cd465bd80f97aafe0f8f3503eda24622cf74ebf433d1f6fae23ba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T09:23:33.778 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 17 09:23:33 smithi105 podman[109993]: 2024-09-17 09:23:33.568437693 +0000 UTC m=+0.548340804 container died 48c3d9e795cd465bd80f97aafe0f8f3503eda24622cf74ebf433d1f6fae23ba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=reef-19e2a2b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T09:23:34.156 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.054+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:34.156 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.054+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:34.156 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.054+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:34.156 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.054+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:34.174 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T09:23:34.214 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:34.214 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T09:23:34.214 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T09:23:34.215 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.7 2024-09-17T09:23:34.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 systemd[1]: Stopping Ceph osd.7 for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:35.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.532+0000 7f0f09269640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:23:35.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.532+0000 7f0f09269640 -1 osd.7 130 *** Got signal Terminated *** 2024-09-17T09:23:35.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:34 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:34.532+0000 7f0f09269640 -1 osd.7 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:23:35.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:35.093+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:35.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:35.093+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:35.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:35.093+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:35.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:35 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:35.093+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:36.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:36.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:36.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:36.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:36.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:36.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:36.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:36 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:36.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:37.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:37.141+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:37.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:37.141+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:37.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:37.141+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:37.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:37 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:37.141+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:38.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:38.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:38.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:38.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:38.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:38.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:38.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:38 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:38.112+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:39.528 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:39.156+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6806 osd.0 since back 2024-09-17T09:22:42.531867+0000 front 2024-09-17T09:22:42.531724+0000 (oldest deadline 2024-09-17T09:23:08.932295+0000) 2024-09-17T09:23:39.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:39.156+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6814 osd.1 since back 2024-09-17T09:22:50.033294+0000 front 2024-09-17T09:22:50.033311+0000 (oldest deadline 2024-09-17T09:23:14.733090+0000) 2024-09-17T09:23:39.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:39.156+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6822 osd.2 since back 2024-09-17T09:22:59.334362+0000 front 2024-09-17T09:22:59.334380+0000 (oldest deadline 2024-09-17T09:23:23.434294+0000) 2024-09-17T09:23:39.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:39.156+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.27:6830 osd.3 since back 2024-09-17T09:23:07.535535+0000 front 2024-09-17T09:23:07.535484+0000 (oldest deadline 2024-09-17T09:23:33.435471+0000) 2024-09-17T09:23:39.529 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7[95911]: 2024-09-17T09:23:39.156+0000 7f0f01875640 -1 osd.7 130 heartbeat_check: no reply from 172.21.15.105:6804 osd.4 since back 2024-09-17T09:23:13.436005+0000 front 2024-09-17T09:23:13.435959+0000 (oldest deadline 2024-09-17T09:23:38.735885+0000) 2024-09-17T09:23:40.135 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:39 smithi105 podman[110094]: 2024-09-17 09:23:39.865780229 +0000 UTC m=+5.435840242 container died 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-19e2a2b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T09:23:40.778 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:40 smithi105 podman[110094]: 2024-09-17 09:23:40.450540584 +0000 UTC m=+6.020600630 container cleanup 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-17T09:23:40.779 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:40 smithi105 bash[110094]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7 2024-09-17T09:23:40.779 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:40 smithi105 podman[110109]: 2024-09-17 09:23:40.668439545 +0000 UTC m=+0.801525418 container remove 3bb4e3186543d7beb157e7ff075d3ba65e1a97d14dc9e18817f90dd02a973922 (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-19e2a2b, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T09:23:41.278 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:40 smithi105 podman[110202]: 2024-09-17 09:23:40.887846963 +0000 UTC m=+0.015473118 image pull 980a0bfcd684674f25c08d857ad8de3b1c8ae738870529edbf9aa463832c8d89 quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9 2024-09-17T09:23:41.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:41 smithi105 podman[110202]: 2024-09-17 09:23:41.009927292 +0000 UTC m=+0.137553438 container create 2e9206a85396bcb636b5c19bdc37ed5c2072666c95cb3c852b00f0f0bc0161ba (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T09:23:41.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:41 smithi105 podman[110202]: 2024-09-17 09:23:41.213379364 +0000 UTC m=+0.341005507 container init 2e9206a85396bcb636b5c19bdc37ed5c2072666c95cb3c852b00f0f0bc0161ba (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-19e2a2b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T09:23:41.279 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:41 smithi105 podman[110202]: 2024-09-17 09:23:41.216672836 +0000 UTC m=+0.344298986 container start 2e9206a85396bcb636b5c19bdc37ed5c2072666c95cb3c852b00f0f0bc0161ba (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=reef-19e2a2b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T09:23:41.683 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:41 smithi105 podman[110202]: 2024-09-17 09:23:41.305626654 +0000 UTC m=+0.433252828 container attach 2e9206a85396bcb636b5c19bdc37ed5c2072666c95cb3c852b00f0f0bc0161ba (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-19e2a2b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T09:23:41.683 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 17 09:23:41 smithi105 podman[110202]: 2024-09-17 09:23:41.400944886 +0000 UTC m=+0.528571037 container died 2e9206a85396bcb636b5c19bdc37ed5c2072666c95cb3c852b00f0f0bc0161ba (image=quay.ceph.io/ceph-ci/ceph@sha256:70b7810862ffedc21315a6c67a84878d4bbd16155279a41a670f6f6be7488da9, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-19e2a2b, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-17T09:23:42.036 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T09:23:42.073 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:42.074 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T09:23:42.074 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-17T09:23:42.074 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a 2024-09-17T09:23:42.278 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 systemd[1]: Stopping Ceph prometheus.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.395Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.396Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.396Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-17T09:23:42.663 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a[64140]: ts=2024-09-17T09:23:42.396Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-17T09:23:42.664 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 podman[110304]: 2024-09-17 09:23:42.405993757 +0000 UTC m=+0.110844796 container died f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T09:23:43.021 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 podman[110304]: 2024-09-17 09:23:42.761675722 +0000 UTC m=+0.466526759 container cleanup f7d88b1000742e320b8153c2574907d86e60aaca934f2642aa6f60402f5aca1c (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-17T09:23:43.021 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 17 09:23:42 smithi105 bash[110304]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-prometheus-a 2024-09-17T09:23:43.197 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-17T09:23:43.230 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:23:43.230 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-17T09:23:43.231 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T09:23:43.395 INFO:teuthology.orchestra.run.smithi027.stderr:usage: cephadm [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR] 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr: [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR] 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr: [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT] 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr: [--retry RETRY] [--env ENV] [--no-container-init] 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr: {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install,registry-login,gather-facts,exporter,host-maintenance,verify-prereqs} 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr: ... 2024-09-17T09:23:43.396 INFO:teuthology.orchestra.run.smithi027.stderr:cephadm: error: unrecognized arguments: --keep-logs 2024-09-17T09:23:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 2 2024-09-17T09:23:43.411 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T09:23:43.554 INFO:teuthology.orchestra.run.smithi105.stderr:usage: cephadm [-h] [--image IMAGE] [--docker] [--data-dir DATA_DIR] 2024-09-17T09:23:43.554 INFO:teuthology.orchestra.run.smithi105.stderr: [--log-dir LOG_DIR] [--logrotate-dir LOGROTATE_DIR] 2024-09-17T09:23:43.555 INFO:teuthology.orchestra.run.smithi105.stderr: [--unit-dir UNIT_DIR] [--verbose] [--timeout TIMEOUT] 2024-09-17T09:23:43.555 INFO:teuthology.orchestra.run.smithi105.stderr: [--retry RETRY] [--env ENV] [--no-container-init] 2024-09-17T09:23:43.555 INFO:teuthology.orchestra.run.smithi105.stderr: {version,pull,inspect-image,ls,list-networks,adopt,rm-daemon,rm-cluster,run,shell,enter,ceph-volume,unit,logs,bootstrap,deploy,check-host,prepare-host,add-repo,rm-repo,install,registry-login,gather-facts,exporter,host-maintenance,verify-prereqs} 2024-09-17T09:23:43.555 INFO:teuthology.orchestra.run.smithi105.stderr: ... 2024-09-17T09:23:43.555 INFO:teuthology.orchestra.run.smithi105.stderr:cephadm: error: unrecognized arguments: --keep-logs 2024-09-17T09:23:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 2 2024-09-17T09:23:43.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:23:43.599 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:23:43.624 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T09:23:43.626 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi027/crash 2024-09-17T09:23:43.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/crash -- . 2024-09-17T09:23:43.675 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi105/crash 2024-09-17T09:23:43.676 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/crash -- . 2024-09-17T09:23:43.722 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T09:23:43.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_STRAY_DAEMON | egrep -v CEPHADM_FAILED_DAEMON | egrep -v CEPHADM_AGENT_DOWN | head -n 1 2024-09-17T09:23:43.751 INFO:teuthology.orchestra.run.smithi027.stderr:grep: /var/log/ceph/d61dd7c6-74d0-11ef-bceb-c7b262605968/ceph.log: No such file or directory 2024-09-17T09:23:43.753 INFO:tasks.cephadm:Compressing logs... 2024-09-17T09:23:43.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:23:43.796 DEBUG:teuthology.orchestra.run.smithi105:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:23:43.819 INFO:teuthology.orchestra.run.smithi027.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-17T09:23:43.820 INFO:teuthology.orchestra.run.smithi105.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-17T09:23:54.413 INFO:tasks.cephadm:Archiving logs... 2024-09-17T09:23:54.414 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi027/log 2024-09-17T09:23:54.415 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T09:23:55.646 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi105/log 2024-09-17T09:23:55.647 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T09:23:56.610 INFO:tasks.cephadm:Removing cluster... 2024-09-17T09:23:56.610 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --force 2024-09-17T09:23:58.339 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:58 smithi027 systemd[1]: Stopping Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:58.612 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:58 smithi027 podman[111908]: 2024-09-17 09:23:58.338349723 +0000 UTC m=+0.122102116 container died 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:23:59.008 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:58 smithi027 podman[111908]: 2024-09-17 09:23:58.71061951 +0000 UTC m=+0.494371898 container cleanup 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:23:59.008 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:58 smithi027 bash[111908]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a 2024-09-17T09:23:59.379 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:59 smithi027 podman[111920]: 2024-09-17 09:23:59.061493722 +0000 UTC m=+0.721678428 container remove 63779f8bce273d57833d2cc12b646d229b7cc5a16e99d8a8d9b0825b4578aa27 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-17T09:23:59.379 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:59 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:23:59.379 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:59 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-17T09:23:59.379 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:59 smithi027 systemd[1]: Stopped Ceph node-exporter.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:23:59.379 INFO:journalctl@ceph.node-exporter.a.smithi027.stdout:Sep 17 09:23:59 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 3.218s CPU time. 2024-09-17T09:23:59.665 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:23:59 smithi027 systemd[1]: Stopping Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:23:59.665 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:23:59 smithi027 ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a[62646]: ts=2024-09-17T09:23:59.623Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-17T09:23:59.949 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:23:59 smithi027 podman[112028]: 2024-09-17 09:23:59.664290054 +0000 UTC m=+0.141152296 container died 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:24:00.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 podman[112028]: 2024-09-17 09:24:00.048083199 +0000 UTC m=+0.524945441 container cleanup 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:24:00.258 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 bash[112028]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a 2024-09-17T09:24:00.546 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 podman[112040]: 2024-09-17 09:24:00.291237733 +0000 UTC m=+0.621083714 container remove 92c6277605b3a70b623eb69742e8a89ebd6ca2f811af04d1fdaf0b59543a39b9 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-17T09:24:00.546 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 podman[112040]: 2024-09-17 09:24:00.36688034 +0000 UTC m=+0.696726275 volume remove 2df0deeaf3fa6e98f759af367c3ac6b91661d7102bde99700a8f46aca7f0ada8 2024-09-17T09:24:00.546 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-17T09:24:00.546 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 systemd[1]: Stopped Ceph alertmanager.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:24:00.546 INFO:journalctl@ceph.alertmanager.a.smithi027.stdout:Sep 17 09:24:00 smithi027 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.563s CPU time. 2024-09-17T09:24:12.533 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d61dd7c6-74d0-11ef-bceb-c7b262605968 --force 2024-09-17T09:24:14.236 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:13 smithi105 systemd[1]: Stopping Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:24:14.528 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:14 smithi105 podman[110806]: 2024-09-17 09:24:14.234988852 +0000 UTC m=+0.111539087 container died 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:24:14.997 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:14 smithi105 podman[110806]: 2024-09-17 09:24:14.624305357 +0000 UTC m=+0.500855599 container cleanup 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:24:14.997 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:14 smithi105 bash[110806]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b 2024-09-17T09:24:14.997 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:14 smithi105 podman[110819]: 2024-09-17 09:24:14.858851162 +0000 UTC m=+0.627384238 container remove 15c01b33b4951b51325c96a8a4d1f3be44f0163b4fea41435d59eab18a7678f0 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-17T09:24:14.998 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:14 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:24:15.278 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 systemd[1]: Stopping Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968... 2024-09-17T09:24:15.278 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:15 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-17T09:24:15.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:15 smithi105 systemd[1]: Stopped Ceph node-exporter.b for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:24:15.279 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 17 09:24:15 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 3.037s CPU time. 2024-09-17T09:24:15.712 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 podman[110927]: 2024-09-17 09:24:15.4475816 +0000 UTC m=+0.111079324 container died 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-type=git, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=) 2024-09-17T09:24:16.028 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 podman[110927]: 2024-09-17 09:24:15.786627958 +0000 UTC m=+0.450125667 container cleanup 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-type=git, version=8.5, io.openshift.expose-services=) 2024-09-17T09:24:16.028 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 bash[110927]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a 2024-09-17T09:24:16.029 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 podman[110939]: 2024-09-17 09:24:15.954550948 +0000 UTC m=+0.507952764 container remove 447ebac1e36ebdc2f6b405e3039d0e30d8f23bccbcd286246d24f95a67e80a5c (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968-grafana-a, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.buildah.version=1.32.0, distribution-scope=public, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-type=git, vendor=Red Hat, Inc., description=Ceph Grafana Container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, version=8.5, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-17T09:24:16.029 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:15 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-17T09:24:16.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:16 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-17T09:24:16.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:16 smithi105 systemd[1]: Stopped Ceph grafana.a for d61dd7c6-74d0-11ef-bceb-c7b262605968. 2024-09-17T09:24:16.528 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 17 09:24:16 smithi105 systemd[1]: ceph-d61dd7c6-74d0-11ef-bceb-c7b262605968@grafana.a.service: Consumed 9.655s CPU time. 2024-09-17T09:24:39.690 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T09:24:39.691 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T09:24:39.712 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T09:24:39.728 INFO:tasks.cephadm:Teardown complete 2024-09-17T09:24:39.728 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T09:24:39.738 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T09:24:39.739 DEBUG:teuthology.orchestra.run.smithi027:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:24:39.754 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:24:39.769 INFO:teuthology.orchestra.run.smithi027.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:24:39.774 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:24:39.775 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2024-09-17T09:24:39.775 INFO:teuthology.orchestra.run.smithi027.stdout:^* hv01.front.sepia.ceph.com 3 6 377 36 -291us[ -270us] +/- 50ms 2024-09-17T09:24:39.775 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 39 -1139us[-1117us] +/- 54ms 2024-09-17T09:24:39.775 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 35 +1867us[+1867us] +/- 75ms 2024-09-17T09:24:39.775 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:24:39.783 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:24:39.786 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:24:39.786 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-17T09:24:39.786 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 40 -151us[ -151us] +/- 50ms 2024-09-17T09:24:39.786 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv02.front.sepia.ceph.com 3 6 377 40 -1021us[-1059us] +/- 54ms 2024-09-17T09:24:39.787 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 +1984us[+1984us] +/- 75ms 2024-09-17T09:24:39.787 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:24:39.787 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T09:24:39.796 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T09:24:39.797 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T09:24:39.805 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T09:24:39.904 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T09:24:39.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T09:24:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:40.012 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-09-17T09:24:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:40.041 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T09:24:40.050 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T09:24:40.099 INFO:teuthology.task.internal:Duration was 2734.102891 seconds 2024-09-17T09:24:40.099 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T09:24:40.110 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T09:24:40.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T09:24:40.112 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T09:24:40.154 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:24:40.160 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:24:40.565 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T09:24:40.565 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2024-09-17T09:24:40.565 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T09:24:40.590 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-09-17T09:24:40.590 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T09:24:40.630 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T09:24:40.630 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:24:40.633 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:24:40.697 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T09:24:40.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T09:24:40.705 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T09:24:41.294 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T09:24:41.304 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T09:24:41.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T09:24:41.330 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T09:24:41.354 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T09:24:41.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:24:41.372 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:24:41.393 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2024-09-17T09:24:41.418 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-09-17T09:24:41.437 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:24:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:41.477 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:24:41.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:41.492 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T09:24:41.501 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T09:24:41.503 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi027 2024-09-17T09:24:41.503 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T09:24:41.565 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_22:08:02-orch-reef-distro-default-smithi/7908238/remote/smithi105 2024-09-17T09:24:41.565 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T09:24:41.605 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T09:24:41.606 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T09:24:41.608 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T09:24:41.653 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T09:24:41.664 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T09:24:41.664 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T09:24:41.673 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T09:24:41.674 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T09:24:41.676 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T09:24:41.689 INFO:teuthology.orchestra.run.smithi027.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 09:24 /home/ubuntu/cephtest 2024-09-17T09:24:41.708 INFO:teuthology.orchestra.run.smithi105.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 09:24 /home/ubuntu/cephtest 2024-09-17T09:24:41.709 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T09:24:41.718 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T09:24:41.788 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} duration: 2734.1028909683228 owner: scheduled_teuthology@teuthology success: true 2024-09-17T09:24:41.788 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T09:24:41.879 INFO:teuthology.run:pass